[Datatable] Sorts null values last (#172691)

## Summary

Closes #169126

We were using the orderBy which sorts the null values at the beginning
on descending order lodash/lodash#4169

I created my own function. Now they are sorted always last like Lens
does.

<img width="1820" alt="image"
src="https://github.com/elastic/kibana/assets/17003240/d6fbd6b9-96fe-4e71-b90f-63a030902cca">

### Checklist

- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

(cherry picked from commit 6be8e5e)

src/plugins/vis_types/table/public/components/table_vis_basic.tsx

@@ -15,14 +15,13 @@ import {
   EuiTitle,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import { orderBy } from 'lodash';
-
 import { IInterpreterRenderHandlers } from '@kbn/expressions-plugin/common';
 import { createTableVisCell } from './table_vis_cell';
 import { TableContext, TableVisConfig, TableVisUseUiStateProps } from '../types';
 import { usePagination } from '../utils';
 import { TableVisControls } from './table_vis_controls';
 import { createGridColumns } from './table_vis_columns';
+import { sortNullsLast } from './utils';
 
 interface TableVisBasicProps {
   fireEvent: IInterpreterRenderHandlers['event'];
@@ -45,13 +44,14 @@ export const TableVisBasic = memo(
     const { columns, rows, formattedColumns } = table;
 
     // custom sorting is in place until the EuiDataGrid sorting gets rid of flaws -> https://github.com/elastic/eui/issues/4108
-    const sortedRows = useMemo(
-      () =>
-        sort.columnIndex !== null && sort.direction
-          ? orderBy(rows, columns[sort.columnIndex]?.id, sort.direction)
-          : rows,
-      [columns, rows, sort]
-    );
+    const sortedRows = useMemo(() => {
+      if (sort.columnIndex !== null && sort.direction) {
+        const id = columns[sort.columnIndex]?.id;
+        return sortNullsLast(rows, sort.direction, id);
+      }
+
+      return rows;
+    }, [columns, rows, sort.columnIndex, sort.direction]);
 
     // renderCellValue is a component which renders a cell based on column and row indexes
     const renderCellValue = useMemo(

src/plugins/vis_types/table/public/components/utils.test.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { sortNullsLast } from './utils';
+
+describe('sortNullsLast', () => {
+  const rows = [
+    {
+      col1: null,
+    },
+    {
+      col1: 'meow',
+    },
+    {
+      col1: 'woof',
+    },
+  ];
+  test('should sort correctly in ascending order', async () => {
+    const sortedRows = sortNullsLast(rows, 'asc', 'col1');
+    expect(sortedRows).toStrictEqual([
+      {
+        col1: 'meow',
+      },
+      {
+        col1: 'woof',
+      },
+      {
+        col1: null,
+      },
+    ]);
+  });
+
+  test('should sort correctly in descending order', async () => {
+    const sortedRows = sortNullsLast(rows, 'desc', 'col1');
+    expect(sortedRows).toStrictEqual([
+      {
+        col1: 'woof',
+      },
+      {
+        col1: 'meow',
+      },
+      {
+        col1: null,
+      },
+    ]);
+  });
+});

src/plugins/vis_types/table/public/components/utils.ts

@@ -7,6 +7,7 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import type { DatatableRow } from '@kbn/expressions-plugin/common';
 import { AggTypes } from '../../common';
 
 const totalAggregations = [
@@ -42,4 +43,31 @@ const totalAggregations = [
   },
 ];
 
-export { totalAggregations };
+const sortNullsLast = (
+  rows: DatatableRow[],
+  direction: 'asc' | 'desc',
+  id: string
+): DatatableRow[] => {
+  return rows.sort((row1, row2) => {
+    const rowA = row1[id];
+    const rowB = row2[id];
+
+    if (rowA === null) {
+      return 1;
+    }
+    if (rowB === null) {
+      return -1;
+    }
+    if (rowA === rowB) {
+      return 0;
+    }
+
+    if (direction === 'desc') {
+      return rowA < rowB ? 1 : -1;
+    } else {
+      return rowA < rowB ? -1 : 1;
+    }
+  });
+};
+
+export { totalAggregations, sortNullsLast };