[ES|QL][Discover] Hide null values from the document viewer (elastic#…

…189601)

## Summary

Closes elastic#188846

Added a switch to the docViewer that hides the fields with null values.
It is visible both in ESQL and dataview mode. The selection is stored in
the localStorage


![meow](https://github.com/user-attachments/assets/3a072081-e41c-482c-8c0f-2028f39a34cd)


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

src/plugins/unified_doc_viewer/public/components/doc_viewer_table/table.tsx

@@ -9,6 +9,7 @@
 import './table.scss';
 import React, { useCallback, useMemo, useState } from 'react';
 import useWindowSize from 'react-use/lib/useWindowSize';
+import useLocalStorage from 'react-use/lib/useLocalStorage';
 import {
   EuiFlexGroup,
   EuiFlexItem,
@@ -22,6 +23,8 @@ import {
   EuiText,
   EuiCallOut,
   useResizeObserver,
+  EuiSwitch,
+  useEuiTheme,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { css } from '@emotion/react';
@@ -72,6 +75,7 @@ const DEFAULT_PAGE_SIZE = 25;
 const PINNED_FIELDS_KEY = 'discover:pinnedFields';
 const PAGE_SIZE = 'discover:pageSize';
 const SEARCH_TEXT = 'discover:searchText';
+const HIDE_NULL_VALUES = 'discover:hideNullValues';
 
 const GRID_COLUMN_FIELD_NAME = 'name';
 const GRID_COLUMN_FIELD_VALUE = 'value';
@@ -135,11 +139,13 @@ export const DocViewerTable = ({
   columnsMeta,
   hit,
   dataView,
+  textBasedHits,
   filter,
   decreaseAvailableHeightBy,
   onAddColumn,
   onRemoveColumn,
 }: DocViewRenderProps) => {
+  const isEsqlMode = Array.isArray(textBasedHits);
   const [containerRef, setContainerRef] = useState<HTMLDivElement | null>(null);
   const { fieldFormats, storage, uiSettings, fieldsMetadata } = getUnifiedDocViewerServices();
   const showMultiFields = uiSettings.get(SHOW_MULTIFIELDS);
@@ -149,6 +155,9 @@ export const DocViewerTable = ({
   const [pinnedFields, setPinnedFields] = useState<string[]>(
     getPinnedFields(currentDataViewId, storage)
   );
+  const [areNullValuesHidden, setAreNullValuesHidden] = useLocalStorage(HIDE_NULL_VALUES, false);
+
+  const { euiTheme } = useEuiTheme();
 
   const flattened = hit.flattened;
   const shouldShowFieldHandler = useMemo(
@@ -265,7 +274,11 @@ export const DocViewerTable = ({
         if (!shouldShowFieldHandler(curFieldName)) {
           return acc;
         }
-
+        const shouldHideNullValue =
+          areNullValuesHidden && flattened[curFieldName] == null && isEsqlMode;
+        if (shouldHideNullValue) {
+          return acc;
+        }
         if (pinnedFields.includes(curFieldName)) {
           acc.pinnedItems.push(fieldToItem(curFieldName, true));
         } else {
@@ -358,6 +371,13 @@ export const DocViewerTable = ({
     [fieldCellActions, fieldValueCellActions, containerWidth]
   );
 
+  const onHideNullValuesChange = useCallback(
+    (e) => {
+      setAreNullValuesHidden(e.target.checked);
+    },
+    [setAreNullValuesHidden]
+  );
+
   const renderCellValue: EuiDataGridProps['renderCellValue'] = useCallback(
     ({ rowIndex, columnId, isDetails }) => {
       const row = rows[rowIndex];
@@ -493,6 +513,25 @@ export const DocViewerTable = ({
           <EuiFlexItem grow={false}>
             <EuiSpacer size="s" />
           </EuiFlexItem>
+          {isEsqlMode && (
+            <EuiFlexItem
+              grow={false}
+              css={css`
+                align-self: end;
+                padding-bottom: ${euiTheme.size.s};
+              `}
+            >
+              <EuiSwitch
+                label={i18n.translate('unifiedDocViewer.hideNullValues.switchLabel', {
+                  defaultMessage: 'Hide fields with null values',
+                })}
+                checked={areNullValuesHidden ?? false}
+                onChange={onHideNullValuesChange}
+                compressed
+                data-test-subj="unifiedDocViewerHideNullValuesSwitch"
+              />
+            </EuiFlexItem>
+          )}
           <EuiFlexItem
             grow={Boolean(containerHeight)}
             css={css`

test/functional/apps/discover/group3/_doc_viewer.ts

@@ -11,11 +11,18 @@ import { FtrProviderContext } from '../ftr_provider_context';
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const kibanaServer = getService('kibanaServer');
-  const PageObjects = getPageObjects(['common', 'discover', 'timePicker', 'header']);
+  const PageObjects = getPageObjects([
+    'common',
+    'discover',
+    'timePicker',
+    'header',
+    'unifiedFieldList',
+  ]);
   const testSubjects = getService('testSubjects');
   const find = getService('find');
   const retry = getService('retry');
   const dataGrid = getService('dataGrid');
+  const monacoEditor = getService('monacoEditor');
 
   describe('discover doc viewer', function describeIndexTests() {
     before(async function () {
@@ -103,5 +110,45 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         });
       });
     });
+
+    describe('hide null values switch - ES|QL mode', function () {
+      beforeEach(async () => {
+        await PageObjects.discover.selectTextBaseLang();
+        await PageObjects.header.waitUntilLoadingHasFinished();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+        await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
+
+        const testQuery = 'from logstash-* | sort @timestamp | limit 10000';
+        await monacoEditor.setCodeEditorValue(testQuery);
+        await testSubjects.click('querySubmitButton');
+        await PageObjects.header.waitUntilLoadingHasFinished();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+        await PageObjects.unifiedFieldList.waitUntilSidebarHasLoaded();
+        await dataGrid.clickRowToggle();
+        await PageObjects.discover.isShowingDocViewer();
+      });
+
+      afterEach(async () => {
+        const fieldSearch = await testSubjects.find('clearSearchButton');
+        await fieldSearch.click(); // clear search
+        const hideNullValuesSwitch = await testSubjects.find(
+          'unifiedDocViewerHideNullValuesSwitch'
+        );
+        await hideNullValuesSwitch.click(); // make sure the switch is off
+      });
+
+      it('should hide fields with null values ', async function () {
+        await PageObjects.discover.findFieldByNameInDocViewer('machine');
+        const results = (await find.allByCssSelector('.kbnDocViewer__fieldName')).length;
+        const hideNullValuesSwitch = await testSubjects.find(
+          'unifiedDocViewerHideNullValuesSwitch'
+        );
+        await hideNullValuesSwitch.click();
+
+        await retry.waitFor('updates', async () => {
+          return (await find.allByCssSelector('.kbnDocViewer__fieldName')).length < results;
+        });
+      });
+    });
   });
 }