Merge branch 'main' into long-numeral-switch

Signed-off-by: Kawika Avilla <[email protected]>

changelogs/fragments/8137.yml

@@ -0,0 +1,2 @@
+refactor:
+- [Workspace] Integrate dashboard admin with dynamic config ([#8137](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8137))

changelogs/fragments/8833.yml

@@ -0,0 +1,2 @@
+fix:
+- Adds test id for workspace multi-deletion button ([#8833](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8833))

changelogs/fragments/8834.yml

@@ -0,0 +1,2 @@
+refactor:
+- [Workspace] Add unit tests for inspect page url ([#8834](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8834))

changelogs/fragments/8839.yml

@@ -0,0 +1,2 @@
+fix:
+- Fix a typo while inspecting values for large numerals in OSD and the JS client ([#8839](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8839))

changelogs/fragments/8848.yml

@@ -0,0 +1,2 @@
+fix:
+- Fix template queries loading and update getSampleQuery interface ([#8848](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8848))

changelogs/fragments/8851.yml

@@ -0,0 +1,3 @@
+feat:
+- Add framework to show banner at the top in discover results canvas ([#8851](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8851))
+- Show indexed views in dataset selector ([#8851](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8851))

changelogs/fragments/8857.yml

@@ -0,0 +1,2 @@
+fix:
+- [Workspace][Bug] Fix inspect page url error. ([#8857](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8857))

changelogs/fragments/8863.yml

@@ -0,0 +1,2 @@
+fix:
+- Keep previous query result if current query result in error ([#8863](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8863))

changelogs/fragments/8866.yml

@@ -0,0 +1,2 @@
+fix:
+- Hide Date Picker for Unsupported Types ([#8866](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8866))

changelogs/fragments/8867.yml

@@ -0,0 +1,2 @@
+fix:
+- Add max height and scroll to error message body ([#8867](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8867))

changelogs/fragments/8871.yml

@@ -0,0 +1,2 @@
+fix:
+- Search on page load out of sync state when clicking submit. ([#8871](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8871))

packages/osd-std/src/json.test.ts

@@ -3,6 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import JSON11 from 'json11';
 import { stringify, parse } from './json';
 
 describe('json', () => {
@@ -90,9 +91,55 @@ describe('json', () => {
     expect(stringify(input, replacer, 2)).toEqual(JSON.stringify(input, replacer, 2));
   });
 
-  it('can handle long numerals while parsing', () => {
-    const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n;
-    const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n;
+  it('can handle positive long numerals while parsing', () => {
+    const longPositiveA = BigInt(Number.MAX_SAFE_INTEGER) * 2n;
+    const longPositiveB = BigInt(Number.MAX_SAFE_INTEGER) * 2n + 1n;
+    const text =
+      `{` +
+      // The space before and after the values, and the lack of spaces before comma are intentional
+      `"\\":${longPositiveA}": "[ ${longPositiveB.toString()}, ${longPositiveA.toString()} ]", ` +
+      `"positive": ${longPositiveA.toString()}, ` +
+      `"array": [ ${longPositiveB.toString()}, ${longPositiveA.toString()} ], ` +
+      `"negative": ${longPositiveB.toString()},` +
+      `"number": 102931203123987` +
+      `}`;
+
+    const result = parse(text);
+    expect(result.positive).toBe(longPositiveA);
+    expect(result.negative).toBe(longPositiveB);
+    expect(result.array).toEqual([longPositiveB, longPositiveA]);
+    expect(result['":' + longPositiveA]).toBe(
+      `[ ${longPositiveB.toString()}, ${longPositiveA.toString()} ]`
+    );
+    expect(result.number).toBe(102931203123987);
+  });
+
+  it('can handle negative long numerals while parsing', () => {
+    const longNegativeA = BigInt(Number.MIN_SAFE_INTEGER) * 2n;
+    const longNegativeB = BigInt(Number.MIN_SAFE_INTEGER) * 2n - 1n;
+    const text =
+      `{` +
+      // The space before and after the values, and the lack of spaces before comma are intentional
+      `"\\":${longNegativeA}": "[ ${longNegativeB.toString()}, ${longNegativeA.toString()} ]", ` +
+      `"positive": ${longNegativeA.toString()}, ` +
+      `"array": [ ${longNegativeB.toString()}, ${longNegativeA.toString()} ], ` +
+      `"negative": ${longNegativeB.toString()},` +
+      `"number": 102931203123987` +
+      `}`;
+
+    const result = parse(text);
+    expect(result.positive).toBe(longNegativeA);
+    expect(result.negative).toBe(longNegativeB);
+    expect(result.array).toEqual([longNegativeB, longNegativeA]);
+    expect(result['":' + longNegativeA]).toBe(
+      `[ ${longNegativeB.toString()}, ${longNegativeA.toString()} ]`
+    );
+    expect(result.number).toBe(102931203123987);
+  });
+
+  it('can handle mixed long numerals while parsing', () => {
+    const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n + 1n;
+    const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n - 1n;
     const text =
       `{` +
       // The space before and after the values, and the lack of spaces before comma are intentional
@@ -113,6 +160,37 @@ describe('json', () => {
     expect(result.number).toBe(102931203123987);
   });
 
+  it('does not use JSON11 when not needed', () => {
+    const spyParse = jest.spyOn(JSON11, 'parse');
+
+    const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n + 1n;
+    const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n - 1n;
+    const text =
+      `{` +
+      `"\\":${longPositive}": "[ ${longNegative.toString()}, ${longPositive.toString()} ]", ` +
+      `"number": 102931203123987` +
+      `}`;
+    parse(text);
+
+    expect(spyParse).not.toHaveBeenCalled();
+  });
+
+  it('uses JSON11 when dealing with long numerals', () => {
+    const spyParse = jest.spyOn(JSON11, 'parse');
+
+    const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n + 1n;
+    const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n - 1n;
+    const text =
+      `{` +
+      `"\\":${longPositive}": "[ ${longNegative.toString()}, ${longPositive.toString()} ]", ` +
+      `"positive": ${longPositive.toString()}, ` +
+      `"number": 102931203123987` +
+      `}`;
+    parse(text);
+
+    expect(spyParse).toHaveBeenCalled();
+  });
+
   it('can handle BigInt values while stringifying', () => {
     const longPositive = BigInt(Number.MAX_SAFE_INTEGER) * 2n;
     const longNegative = BigInt(Number.MIN_SAFE_INTEGER) * 2n;

packages/osd-std/src/json.ts

@@ -69,7 +69,7 @@ export const parse = (
       numeralsAreNumbers &&
       typeof val === 'number' &&
       isFinite(val) &&
-      (val < Number.MAX_SAFE_INTEGER || val > Number.MAX_SAFE_INTEGER)
+      (val < Number.MIN_SAFE_INTEGER || val > Number.MAX_SAFE_INTEGER)
     ) {
       numeralsAreNumbers = false;
     }

scripts/postinstall.js

@@ -84,6 +84,15 @@ const run = async () => {
       },
     ])
   );
+  //ToDo: Remove when opensearch-js is released to include https://github.com/opensearch-project/opensearch-js/pull/889
+  promises.push(
+    patchFile('node_modules/@opensearch-project/opensearch/lib/Serializer.js', [
+      {
+        from: 'val < Number.MAX_SAFE_INTEGER',
+        to: 'val < Number.MIN_SAFE_INTEGER',
+      },
+    ])
+  );
 
   await Promise.all(promises);
 };

src/dev/generate_release_note.ts

@@ -3,6 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { ToolingLog } from '@osd/dev-utils';
 import { join, resolve } from 'path';
 import { readFileSync, writeFileSync, Dirent, rm, rename, promises as fsPromises } from 'fs';
 import { load as loadYaml } from 'js-yaml';
@@ -19,6 +20,11 @@ import {
   filePath,
 } from './generate_release_note_helper';
 
+const log = new ToolingLog({
+  level: 'info',
+  writeTo: process.stdout,
+});
+
 // Function to add content after the 'Unreleased' section in the changelog
 function addContentAfterUnreleased(path: string, newContent: string): void {
   let fileContent = readFileSync(path, 'utf8');
@@ -60,35 +66,63 @@ async function readFragments() {
   ) as unknown) as Changelog;
 
   const fragmentPaths = await readdir(fragmentDirPath, { withFileTypes: true });
+  const failedFragments: string[] = [];
+
   for (const fragmentFilename of fragmentPaths) {
     // skip non yml or yaml files
     if (!/\.ya?ml$/i.test(fragmentFilename.name)) {
-      // eslint-disable-next-line no-console
-      console.warn(`Skipping non yml or yaml file ${fragmentFilename.name}`);
+      log.info(`Skipping non yml or yaml file ${fragmentFilename.name}`);
       continue;
     }
 
-    const fragmentPath = join(fragmentDirPath, fragmentFilename.name);
-    const fragmentContents = readFileSync(fragmentPath, { encoding: 'utf-8' });
-
-    validateFragment(fragmentContents);
-
-    const fragmentContentLines = fragmentContents.split('\n');
-    // Adding a quotes to the second line and escaping exisiting " within the line
-    fragmentContentLines[1] = fragmentContentLines[1].replace(/-\s*(.*)/, (match, p1) => {
-      // Escape any existing quotes in the content
-      const escapedContent = p1.replace(/"/g, '\\"');
-      return `- "${escapedContent}"`;
-    });
-
-    const processedFragmentContent = fragmentContentLines.join('\n');
-
-    const fragmentYaml = loadYaml(processedFragmentContent) as Changelog;
-    for (const [sectionKey, entries] of Object.entries(fragmentYaml)) {
-      sections[sectionKey as SectionKey].push(...entries);
+    try {
+      const fragmentPath = join(fragmentDirPath, fragmentFilename.name);
+      const fragmentContents = readFileSync(fragmentPath, { encoding: 'utf-8' });
+
+      try {
+        validateFragment(fragmentContents);
+      } catch (validationError) {
+        log.info(`Validation failed for ${fragmentFilename.name}: ${validationError.message}`);
+        failedFragments.push(
+          `${fragmentFilename.name} (Validation Error: ${validationError.message})`
+        );
+        continue;
+      }
+
+      const fragmentContentLines = fragmentContents.split('\n');
+      // Adding a quotes to the second line and escaping existing " within the line
+      fragmentContentLines[1] = fragmentContentLines[1].replace(/-\s*(.*)/, (match, p1) => {
+        // Escape any existing quotes in the content
+        const escapedContent = p1.replace(/"/g, '\\"');
+        return `- "${escapedContent}"`;
+      });
+
+      const processedFragmentContent = fragmentContentLines.join('\n');
+
+      try {
+        const fragmentYaml = loadYaml(processedFragmentContent) as Changelog;
+        for (const [sectionKey, entries] of Object.entries(fragmentYaml)) {
+          sections[sectionKey as SectionKey].push(...entries);
+        }
+      } catch (yamlError) {
+        log.info(`Failed to parse YAML in ${fragmentFilename.name}: ${yamlError.message}`);
+        failedFragments.push(`${fragmentFilename.name} (YAML Parse Error: ${yamlError.message})`);
+        continue;
+      }
+    } catch (error) {
+      log.info(`Failed to process ${fragmentFilename.name}: ${error.message}`);
+      failedFragments.push(`${fragmentFilename.name} (Processing Error: ${error.message})`);
+      continue;
     }
   }
-  return { sections, fragmentPaths };
+
+  if (failedFragments.length > 0) {
+    log.info('\nThe following changelog fragments were skipped due to errors:');
+    failedFragments.forEach((fragment) => log.info(`- ${fragment}`));
+    log.info('\nPlease review and fix these fragments for inclusion in the next release.\n');
+  }
+
+  return { sections, fragmentPaths, failedFragments };
 }
 
 async function moveFragments(fragmentPaths: Dirent[], fragmentTempDirPath: string): Promise<void> {
@@ -128,16 +162,22 @@ function generateReleaseNote(changelogSections: string[]) {
 }
 
 (async () => {
-  const { sections, fragmentPaths } = await readFragments();
-  // create folder for temp fragments
-  const fragmentTempDirPath = await fsPromises.mkdtemp(join(fragmentDirPath, 'tmp_fragments-'));
-  // move fragments to temp fragments folder
-  await moveFragments(fragmentPaths, fragmentTempDirPath);
+  const { sections, fragmentPaths, failedFragments } = await readFragments();
 
-  const changelogSections = generateChangelog(sections);
+  // Only proceed if we have some valid fragments
+  if (Object.values(sections).some((section) => section.length > 0)) {
+    // create folder for temp fragments
+    const fragmentTempDirPath = await fsPromises.mkdtemp(join(fragmentDirPath, 'tmp_fragments-'));
+    // move fragments to temp fragments folder
+    await moveFragments(fragmentPaths, fragmentTempDirPath);
 
-  generateReleaseNote(changelogSections);
+    const changelogSections = generateChangelog(sections);
+    generateReleaseNote(changelogSections);
 
-  // remove temp fragments folder
-  await deleteFragments(fragmentTempDirPath);
+    // remove temp fragments folder
+    await deleteFragments(fragmentTempDirPath);
+  } else {
+    log.error('No valid changelog entries were found. Release notes generation aborted.');
+    process.exit(1);
+  }
 })();

src/plugins/data/common/datasets/types.ts

@@ -247,6 +247,13 @@ export interface Dataset extends BaseDataset {
   timeFieldName?: string;
   /** Optional language to default to from the language selector */
   language?: string;
+  /** Optional reference to the source dataset. Example usage is for indexed views to store the
+   * reference to the table dataset
+   */
+  sourceDatasetRef?: {
+    id: string;
+    type: string;
+  };
 }
 
 export interface DatasetField {

src/plugins/data/public/antlr/opensearch_ppl/.generated/OpenSearchPPLLexer.ts

@@ -1,4 +1,4 @@
-// Generated from grammar/OpenSearchPPLLexer.g4 by ANTLR 4.13.1
+// Generated from ./src/plugins/data/public/antlr/opensearch_ppl/grammar/OpenSearchPPLLexer.g4 by ANTLR 4.13.1
 
 import * as antlr from "antlr4ng";
 import { Token } from "antlr4ng";