[Security Solution] Adapt special character escape according to kuery…

… grammer (#198288)

## Summary

As mentioned in
https://github.com/elastic/kibana/security/code-scanning/349, this PR
resolves the escaping issue.

Additionally, it also adds more candiadates for escaping as mentioned in
`kuery` grammar as shown below .


https://github.com/elastic/kibana/blob/d6b4fe9e6ee2e18691b0fa818c4e2c4ed3c413fc/packages/kbn-es-query/grammar/grammar.peggy#L295-L298

## Solution

This PR replicates #128289 in
`7.17` where `escape_kquery` was moved from `data` plugin to `es-query`
package. This should have been backported to `7.17` but was not.

packages/kbn-es-query/src/kuery/index.ts

@@ -24,3 +24,4 @@ export { KQLSyntaxError } from './kuery_syntax_error';
 export { nodeTypes, nodeBuilder } from './node_types';
 export { fromKueryExpression } from './ast';
 export type { DslQuery, KueryNode, KueryQueryOptions } from './types';
+export { escapeKuery } from './utils/escape_kuery';

packages/kbn-es-query/src/kuery/utils/escape_kuery.test.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { escapeQuotes, escapeKuery } from './escape_kuery';
+
+describe('Kuery escape', () => {
+  test('should escape quotes', () => {
+    const value = 'I said, "Hello."';
+    const expected = 'I said, \\"Hello.\\"';
+
+    expect(escapeQuotes(value)).toBe(expected);
+  });
+
+  test('should escape backslashes and quotes', () => {
+    const value = 'Backslashes \\" in the middle and ends with quotes \\"';
+    const expected = 'Backslashes \\\\\\" in the middle and ends with quotes \\\\\\"';
+
+    expect(escapeQuotes(value)).toBe(expected);
+  });
+
+  test('should escape special characters', () => {
+    const value = `This \\ has (a lot of) <special> characters, don't you *think*? "Yes."`;
+    const expected = `This \\\\ has \\(a lot of\\) \\<special\\> characters, don't you \\*think\\*? \\"Yes.\\"`;
+
+    expect(escapeKuery(value)).toBe(expected);
+  });
+
+  test('should escape keywords', () => {
+    const value = 'foo and bar or baz not qux';
+    const expected = 'foo \\and bar \\or baz \\not qux';
+
+    expect(escapeKuery(value)).toBe(expected);
+  });
+
+  test('should escape keywords next to each other', () => {
+    const value = 'foo and bar or not baz';
+    const expected = 'foo \\and bar \\or \\not baz';
+
+    expect(escapeKuery(value)).toBe(expected);
+  });
+
+  test('should not escape keywords without surrounding spaces', () => {
+    const value = 'And this has keywords, or does it not?';
+    const expected = 'And this has keywords, \\or does it not?';
+
+    expect(escapeKuery(value)).toBe(expected);
+  });
+
+  test('should escape uppercase keywords', () => {
+    const value = 'foo AND bar';
+    const expected = 'foo \\AND bar';
+
+    expect(escapeKuery(value)).toBe(expected);
+  });
+
+  test('should escape both keywords and special characters', () => {
+    const value = 'Hello, world, and <nice> to meet you!';
+    const expected = 'Hello, world, \\and \\<nice\\> to meet you!';
+
+    expect(escapeKuery(value)).toBe(expected);
+  });
+
+  test('should escape newlines and tabs', () => {
+    const value = 'This\nhas\tnewlines\r\nwith\ttabs';
+    const expected = 'This\\nhas\\tnewlines\\r\\nwith\\ttabs';
+
+    expect(escapeKuery(value)).toBe(expected);
+  });
+});

packages/kbn-es-query/src/kuery/utils/escape_kuery.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { flow } from 'lodash';
+
+/**
+ * Escapes backslashes and double-quotes. (Useful when putting a string in quotes to use as a value
+ * in a KQL expression. See the QuotedCharacter rule in kuery.peg.)
+ */
+export function escapeQuotes(str: string) {
+  return str.replace(/[\\"]/g, '\\$&');
+}
+
+/**
+ * Escapes a Kuery node value to ensure that special characters, operators, and whitespace do not result in a parsing error or unintended
+ * behavior when using the value as an argument for the `buildNode` function.
+ */
+export const escapeKuery = flow(escapeSpecialCharacters, escapeAndOr, escapeNot, escapeWhitespace);
+
+// See the SpecialCharacter rule in kuery.peg
+function escapeSpecialCharacters(str: string) {
+  return str.replace(/[\\():<>"*]/g, '\\$&'); // $& means the whole matched string
+}
+
+// See the Keyword rule in kuery.peg
+function escapeAndOr(str: string) {
+  return str.replace(/(\s+)(and|or)(\s+)/gi, '$1\\$2$3');
+}
+
+function escapeNot(str: string) {
+  return str.replace(/not(\s+)/gi, '\\$&');
+}
+
+// See the Space rule in kuery.peg
+function escapeWhitespace(str: string) {
+  return str.replace(/\t/g, '\\t').replace(/\r/g, '\\r').replace(/\n/g, '\\n');
+}

src/plugins/data/public/autocomplete/providers/kql_query_suggestion/lib/escape_kuery.test.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { escapeQuotes, escapeKuery } from './escape_kuery';
+import { escapeQuotes } from './escape_kuery';
 
 describe('Kuery escape', () => {
   test('should escape quotes', () => {
@@ -22,53 +22,4 @@ describe('Kuery escape', () => {
 
     expect(escapeQuotes(value)).toBe(expected);
   });
-
-  test('should escape special characters', () => {
-    const value = `This \\ has (a lot of) <special> characters, don't you *think*? "Yes."`;
-    const expected = `This \\\\ has \\(a lot of\\) \\<special\\> characters, don't you \\*think\\*? \\"Yes.\\"`;
-
-    expect(escapeKuery(value)).toBe(expected);
-  });
-
-  test('should escape keywords', () => {
-    const value = 'foo and bar or baz not qux';
-    const expected = 'foo \\and bar \\or baz \\not qux';
-
-    expect(escapeKuery(value)).toBe(expected);
-  });
-
-  test('should escape keywords next to each other', () => {
-    const value = 'foo and bar or not baz';
-    const expected = 'foo \\and bar \\or \\not baz';
-
-    expect(escapeKuery(value)).toBe(expected);
-  });
-
-  test('should not escape keywords without surrounding spaces', () => {
-    const value = 'And this has keywords, or does it not?';
-    const expected = 'And this has keywords, \\or does it not?';
-
-    expect(escapeKuery(value)).toBe(expected);
-  });
-
-  test('should escape uppercase keywords', () => {
-    const value = 'foo AND bar';
-    const expected = 'foo \\AND bar';
-
-    expect(escapeKuery(value)).toBe(expected);
-  });
-
-  test('should escape both keywords and special characters', () => {
-    const value = 'Hello, world, and <nice> to meet you!';
-    const expected = 'Hello, world, \\and \\<nice\\> to meet you!';
-
-    expect(escapeKuery(value)).toBe(expected);
-  });
-
-  test('should escape newlines and tabs', () => {
-    const value = 'This\nhas\tnewlines\r\nwith\ttabs';
-    const expected = 'This\\nhas\\tnewlines\\r\\nwith\\ttabs';
-
-    expect(escapeKuery(value)).toBe(expected);
-  });
 });

src/plugins/data/public/autocomplete/providers/kql_query_suggestion/lib/escape_kuery.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { flow } from 'lodash';
+import { escapeKuery } from '@kbn/es-query';
 
 /**
  * Escapes backslashes and double-quotes. (Useful when putting a string in quotes to use as a value
@@ -16,23 +16,5 @@ export function escapeQuotes(str: string) {
   return str.replace(/[\\"]/g, '\\$&');
 }
 
-export const escapeKuery = flow(escapeSpecialCharacters, escapeAndOr, escapeNot, escapeWhitespace);
-
-// See the SpecialCharacter rule in kuery.peg
-function escapeSpecialCharacters(str: string) {
-  return str.replace(/[\\():<>"*]/g, '\\$&'); // $& means the whole matched string
-}
-
-// See the Keyword rule in kuery.peg
-function escapeAndOr(str: string) {
-  return str.replace(/(\s+)(and|or)(\s+)/gi, '$1\\$2$3');
-}
-
-function escapeNot(str: string) {
-  return str.replace(/not(\s+)/gi, '\\$&');
-}
-
-// See the Space rule in kuery.peg
-function escapeWhitespace(str: string) {
-  return str.replace(/\t/g, '\\t').replace(/\r/g, '\\r').replace(/\n/g, '\\n');
-}
+// Re-export this function from the @kbn/es-query package to avoid refactoring
+export { escapeKuery };

x-pack/plugins/security_solution/public/common/lib/keury/index.test.ts

@@ -6,60 +6,74 @@
  */
 
 import expect from '@kbn/expect';
-import { escapeKuery } from '.';
+import { escapeQueryValue } from '.';
 
-describe('Kuery escape', () => {
-  it('should not remove white spaces quotes', () => {
-    const value = ' netcat';
-    const expected = ' netcat';
-    expect(escapeKuery(value)).to.be(expected);
-  });
+const TEST_QUERIES = [
+  {
+    description: 'should not remove white spaces quotes',
+    value: ' netcat',
+    expected: ' netcat',
+  },
+  {
+    description: 'should escape quotes',
+    value: 'I said, "Hello."',
+    expected: 'I said, \\"Hello.\\"',
+  },
+  {
+    description: 'should escape special characters',
+    value: `This \\ has (a lot of) <special> characters, don't you *think*? "Yes."`,
+    expected: `This \\\\ has \\(a lot of\\) \\<special\\> characters, don't you \\*think\\*? \\"Yes.\\"`,
+  },
+  {
+    description: 'should escape keywords',
+    value: 'foo and bar or baz not qux',
+    expected: 'foo \\and bar \\or baz \\not qux',
+  },
+  {
+    description: 'should escape keywords next to each other',
+    value: 'foo and bar or not baz',
+    expected: 'foo \\and bar \\or \\not baz',
+  },
+  {
+    description: 'should NOT escape keywords without surrounding spaces',
+    value: 'And this has keywords, or does it not?',
+    expected: 'And this has keywords, \\or does it not?',
+  },
+  {
+    description: 'should escape uppercase keywords',
+    value: 'foo AND bar',
+    expected: 'foo \\AND bar',
+  },
+  {
+    description: 'should escape special characters and NOT keywords',
+    value: 'Hello, "world", and <nice> to meet you!',
+    expected: 'Hello, \\"world\\", \\and \\<nice\\> to meet you!',
+  },
+  {
+    description: 'should escape newlines and tabs',
+    value: 'This\nhas\tnewlines\r\nwith\ttabs',
+    expected: 'This\\nhas\\tnewlines\\r\\nwith\\ttabs',
+  },
+  {
+    description: 'should escape backslashes',
+    value: 'This\\has\\backslashes',
+    expected: 'This\\\\has\\\\backslashes',
+  },
+  {
+    description: 'should escape multiple backslashes and quotes',
+    value: 'This\\ has 2" quotes & \\ 2 "backslashes',
+    expected: 'This\\\\ has 2\\" quotes & \\\\ 2 \\"backslashes',
+  },
+  {
+    description: 'should escape all special character according to kuery.peg SpecialCharacter rule',
+    value: '\\():"*',
+    expected: '\\\\\\(\\)\\:\\"\\*',
+  },
+];
 
-  it('should escape quotes', () => {
-    const value = 'I said, "Hello."';
-    const expected = 'I said, \\"Hello.\\"';
-    expect(escapeKuery(value)).to.be(expected);
-  });
-
-  it('should escape special characters', () => {
-    const value = `This \\ has (a lot of) <special> characters, don't you *think*? "Yes."`;
-    const expected = `This \\ has (a lot of) <special> characters, don't you *think*? \\"Yes.\\"`;
-    expect(escapeKuery(value)).to.be(expected);
-  });
-
-  it('should NOT escape keywords', () => {
-    const value = 'foo and bar or baz not qux';
-    const expected = 'foo and bar or baz not qux';
-    expect(escapeKuery(value)).to.be(expected);
-  });
-
-  it('should NOT escape keywords next to each other', () => {
-    const value = 'foo and bar or not baz';
-    const expected = 'foo and bar or not baz';
-    expect(escapeKuery(value)).to.be(expected);
-  });
-
-  it('should not escape keywords without surrounding spaces', () => {
-    const value = 'And this has keywords, or does it not?';
-    const expected = 'And this has keywords, or does it not?';
-    expect(escapeKuery(value)).to.be(expected);
-  });
-
-  it('should NOT escape uppercase keywords', () => {
-    const value = 'foo AND bar';
-    const expected = 'foo AND bar';
-    expect(escapeKuery(value)).to.be(expected);
-  });
-
-  it('should escape special characters and NOT keywords', () => {
-    const value = 'Hello, "world", and <nice> to meet you!';
-    const expected = 'Hello, \\"world\\", and <nice> to meet you!';
-    expect(escapeKuery(value)).to.be(expected);
-  });
-
-  it('should escape newlines and tabs', () => {
-    const value = 'This\nhas\tnewlines\r\nwith\ttabs';
-    const expected = 'This\\nhas\\tnewlines\\r\\nwith\\ttabs';
-    expect(escapeKuery(value)).to.be(expected);
+describe('Kuery escape', () => {
+  it.each(TEST_QUERIES)('$description', ({ description, value, expected }) => {
+    const result = escapeQueryValue(value);
+    expect(result).to.be(`"${expected}"`);
   });
 });

x-pack/plugins/security_solution/public/common/lib/keury/index.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { isEmpty, isString, flow } from 'lodash/fp';
+import { isEmpty, isString } from 'lodash/fp';
 
 import {
   EsQueryConfig,
@@ -15,6 +15,7 @@ import {
   toElasticsearchQuery,
   fromKueryExpression,
   IndexPatternBase,
+  escapeKuery,
 } from '@kbn/es-query';
 
 export const convertKueryToElasticSearchQuery = (
@@ -54,20 +55,6 @@ export const escapeQueryValue = (val: number | string = ''): string | number =>
   return val;
 };
 
-const escapeWhitespace = (val: string) =>
-  val.replace(/\t/g, '\\t').replace(/\r/g, '\\r').replace(/\n/g, '\\n');
-
-// See the SpecialCharacter rule in kuery.peg
-const escapeSpecialCharacters = (val: string) => val.replace(/["]/g, '\\$&'); // $& means the whole matched string
-
-// See the Keyword rule in kuery.peg
-// I do not think that we need that anymore since we are doing a full match_phrase all the time now => return `"${escapeKuery(val)}"`;
-// const escapeAndOr = (val: string) => val.replace(/(\s+)(and|or)(\s+)/gi, '$1\\$2$3');
-
-// const escapeNot = (val: string) => val.replace(/not(\s+)/gi, '\\$&');
-
-export const escapeKuery = flow(escapeSpecialCharacters, escapeWhitespace);
-
 export const convertToBuildEsQuery = ({
   config,
   indexPattern,