[Test] Add field list cache busting for functional tests

docs/concepts/data-views.asciidoc

@@ -172,3 +172,14 @@ WARNING: Deleting a {data-source} breaks all visualizations, saved searches, and
 
 . Find the {data-source} that you want to delete, and then
 click image:management/index-patterns/images/delete.png[Delete icon] in the *Actions* column.
+
+[float]
+[[data-view-field-cache]]
+=== {data-source} field cache
+
+The browser caches {data-source} field lists for increased performance. This is particularly impactful 
+for {data-sources} with a high field count that span a large number of indices and clusters. The field 
+list is updated every couple of minutes in typical {kib} usage. Alternatively, use the refresh button on the {data-source} 
+management detail page to get an updated field list. A force reload of {kib} has the same effect. 
+
+The field list may be impacted by changes in indices and user permissions.

packages/core/http/core-http-router-server-internal/src/response.ts

@@ -17,6 +17,7 @@ import type {
   ErrorHttpResponseOptions,
   KibanaErrorResponseFactory,
   KibanaRedirectionResponseFactory,
+  KibanaNotModifiedResponseFactory,
   KibanaSuccessResponseFactory,
   KibanaResponseFactory,
   LifecycleResponseFactory,
@@ -51,6 +52,10 @@ const redirectionResponseFactory: KibanaRedirectionResponseFactory = {
   redirected: (options: RedirectResponseOptions) => new KibanaResponse(302, options.body, options),
 };
 
+const notModifiedResponseFactory: KibanaNotModifiedResponseFactory = {
+  notModified: (options: HttpResponseOptions = {}) => new KibanaResponse(304, undefined, options),
+};
+
 const errorResponseFactory: KibanaErrorResponseFactory = {
   badRequest: (options: ErrorHttpResponseOptions = {}) =>
     new KibanaResponse(400, options.body || 'Bad Request', options),
@@ -120,6 +125,7 @@ export const fileResponseFactory = {
 export const kibanaResponseFactory: KibanaResponseFactory = {
   ...successResponseFactory,
   ...redirectionResponseFactory,
+  ...notModifiedResponseFactory,
   ...errorResponseFactory,
   ...fileResponseFactory,
   custom: <T extends HttpResponsePayload | ResponseError>(

packages/core/http/core-http-router-server-internal/src/response_adapter.ts

@@ -33,7 +33,8 @@ function setHeaders(response: HapiResponseObject, headers: Record<string, string
 
 const statusHelpers = {
   isSuccess: (code: number) => code >= 100 && code < 300,
-  isRedirect: (code: number) => code >= 300 && code < 400,
+  isNotModified: (code: number) => code === 304,
+  isRedirect: (code: number) => code >= 300 && code < 400 && code !== 304,
   isError: (code: number) => code >= 400 && code < 600,
 };
 
@@ -76,7 +77,10 @@ export class HapiResponseAdapter {
     if (statusHelpers.isError(kibanaResponse.status)) {
       return this.toError(kibanaResponse);
     }
-    if (statusHelpers.isSuccess(kibanaResponse.status)) {
+    if (
+      statusHelpers.isSuccess(kibanaResponse.status) ||
+      statusHelpers.isNotModified(kibanaResponse.status)
+    ) {
       return this.toSuccess(kibanaResponse);
     }
     if (statusHelpers.isRedirect(kibanaResponse.status)) {

packages/core/http/core-http-router-server-mocks/src/router.mock.ts

@@ -121,6 +121,7 @@ const createResponseFactoryMock = (): jest.Mocked<KibanaResponseFactory> => ({
   ok: jest.fn(),
   accepted: jest.fn(),
   noContent: jest.fn(),
+  notModified: jest.fn(),
   custom: jest.fn(),
   redirected: jest.fn(),
   badRequest: jest.fn(),

packages/core/http/core-http-server/index.ts

@@ -95,6 +95,7 @@ export type {
   IKibanaSocket,
   KibanaErrorResponseFactory,
   KibanaRedirectionResponseFactory,
+  KibanaNotModifiedResponseFactory,
   KibanaSuccessResponseFactory,
   KibanaResponseFactory,
   LifecycleResponseFactory,

packages/core/http/core-http-server/src/router/index.ts

@@ -66,6 +66,7 @@ export type { IKibanaSocket } from './socket';
 export type {
   KibanaErrorResponseFactory,
   KibanaRedirectionResponseFactory,
+  KibanaNotModifiedResponseFactory,
   KibanaSuccessResponseFactory,
   KibanaResponseFactory,
   LifecycleResponseFactory,

packages/core/http/core-http-server/src/router/response.ts

@@ -95,9 +95,7 @@ export interface FileHttpResponseOptions<T extends HttpResponsePayload | Respons
  * @public
  */
 export type RedirectResponseOptions = HttpResponseOptions & {
-  headers: {
-    location: string;
-  };
+  headers?: ResponseHeaders;
 };
 
 /**

packages/core/http/core-http-server/src/router/response_factory.ts

@@ -61,6 +61,18 @@ export interface KibanaRedirectionResponseFactory {
   redirected(options: RedirectResponseOptions): IKibanaResponse;
 }
 
+/**
+ * @public
+ */
+export interface KibanaNotModifiedResponseFactory {
+  /**
+   * Content not modified.
+   * Status code: `304`.
+   * @param options - {@link HttpResponseOptions} configures HTTP response body & headers.
+   */
+  notModified(options: HttpResponseOptions): IKibanaResponse;
+}
+
 /**
  * @public
  */
@@ -200,6 +212,7 @@ export interface KibanaErrorResponseFactory {
  */
 export type KibanaResponseFactory = KibanaSuccessResponseFactory &
   KibanaRedirectionResponseFactory &
+  KibanaNotModifiedResponseFactory &
   KibanaErrorResponseFactory & {
     /**
      * Creates a response with defined status code and payload.

packages/kbn-optimizer/limits.yml

@@ -34,7 +34,7 @@ pageLoadAssetSize:
   dataViewEditor: 28082
   dataViewFieldEditor: 27000
   dataViewManagement: 5100
-  dataViews: 48300
+  dataViews: 50000
   dataVisualizer: 27530
   devTools: 38637
   discover: 99999

src/plugins/data_view_management/public/components/edit_index_pattern/edit_index_pattern.tsx

@@ -95,6 +95,9 @@ export const EditIndexPattern = withRouter(
     const [showEditDialog, setShowEditDialog] = useState<boolean>(false);
     const [relationships, setRelationships] = useState<SavedObjectRelationWithTitle[]>([]);
     const [allowedTypes, setAllowedTypes] = useState<SavedObjectManagementTypeInfo[]>([]);
+    const [refreshCount, setRefreshCount] = useState<number>(0); // used for forcing rerender of field list
+    const [isRefreshing, setIsRefreshing] = React.useState(false);
+
     const conflictFieldsUrl = useMemo(() => {
       return setStateToKbnUrl(
         APP_STATE_STORAGE_KEY,
@@ -144,7 +147,7 @@ export const EditIndexPattern = withRouter(
       setConflictedFields(
         indexPattern.fields.getAll().filter((field) => field.type === 'conflict')
       );
-    }, [indexPattern]);
+    }, [indexPattern, refreshCount]);
 
     useEffect(() => {
       setTags(
@@ -246,8 +249,15 @@ export const EditIndexPattern = withRouter(
           deleteIndexPatternClick={() =>
             removeHandler([indexPattern as RemoveDataViewProps], <div>{warning}</div>)
           }
+          refreshIndexPatternClick={async () => {
+            setIsRefreshing(true);
+            await dataViews.refreshFields(indexPattern, false, true);
+            setRefreshCount(refreshCount + 1); // rerender field list
+            setIsRefreshing(false);
+          }}
           defaultIndex={defaultIndex}
           canSave={userEditPermission}
+          isRefreshing={isRefreshing}
         >
           <EuiHorizontalRule margin="none" />
           <EuiSpacer size="l" />

src/plugins/data_view_management/public/components/edit_index_pattern/index_header/index_header.tsx

@@ -8,7 +8,7 @@
 
 import React from 'react';
 import { i18n } from '@kbn/i18n';
-import { EuiButton, EuiButtonEmpty, EuiPageHeader } from '@elastic/eui';
+import { EuiButton, EuiButtonEmpty, EuiPageHeader, EuiToolTip } from '@elastic/eui';
 import { DataView } from '@kbn/data-views-plugin/public';
 
 interface IndexHeaderProps {
@@ -17,7 +17,9 @@ interface IndexHeaderProps {
   setDefault?: () => void;
   editIndexPatternClick?: () => void;
   deleteIndexPatternClick?: () => void;
+  refreshIndexPatternClick?: () => void;
   canSave: boolean;
+  isRefreshing?: boolean;
 }
 
 const setDefaultAriaLabel = i18n.translate('indexPatternManagement.editDataView.setDefaultAria', {
@@ -44,14 +46,32 @@ const removeTooltip = i18n.translate('indexPatternManagement.editDataView.remove
   defaultMessage: 'Delete',
 });
 
+const refreshAriaLabel = i18n.translate('indexPatternManagement.editDataView.refreshAria', {
+  defaultMessage: 'Refresh',
+});
+
+const refreshTooltip = i18n.translate('indexPatternManagement.editDataView.refreshTooltip', {
+  defaultMessage: 'Refresh local copy of data view field list',
+});
+
+const refreshLabel = i18n.translate('indexPatternManagement.editDataView.refreshLabel', {
+  defaultMessage: 'Refresh',
+});
+
+const isLoadingLabel = i18n.translate('indexPatternManagement.editDataView.refreshLoading', {
+  defaultMessage: 'Loading...',
+});
+
 export const IndexHeader: React.FC<IndexHeaderProps> = ({
   defaultIndex,
   indexPattern,
   setDefault,
   editIndexPatternClick,
   deleteIndexPatternClick,
+  refreshIndexPatternClick,
   children,
   canSave,
+  isRefreshing,
 }) => {
   return (
     <EuiPageHeader
@@ -63,10 +83,23 @@ export const IndexHeader: React.FC<IndexHeaderProps> = ({
             iconType="pencil"
             aria-label={editAriaLabel}
             data-test-subj="editIndexPatternButton"
+            color="primary"
           >
             {editTooltip}
           </EuiButton>
         ),
+        <EuiToolTip content={<p>{refreshTooltip}</p>}>
+          <EuiButtonEmpty
+            onClick={refreshIndexPatternClick}
+            iconType="refresh"
+            aria-label={refreshAriaLabel}
+            data-test-subj="refreshDataViewButton"
+            isLoading={isRefreshing}
+            isDisabled={isRefreshing}
+          >
+            {isRefreshing ? isLoadingLabel : refreshLabel}
+          </EuiButtonEmpty>
+        </EuiToolTip>,
         defaultIndex !== indexPattern.id && setDefault && canSave && indexPattern.isPersisted() && (
           <EuiButton
             onClick={setDefault}

src/plugins/data_view_management/public/components/edit_index_pattern/tabs/tabs.tsx

@@ -284,7 +284,9 @@ export const Tabs: React.FC<TabsProps> = ({
 
     setIndexedFieldTypes(convertToEuiFilterOptions(tempIndexedFieldTypes));
     setScriptedFieldLanguages(convertToEuiFilterOptions(tempScriptedFieldLanguages));
-  }, [indexPattern]);
+    // need to reset based on changes to fields but indexPattern is the same
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [indexPattern, fields]);
 
   const closeFieldEditor = useCallback(() => {
     if (closeEditorHandler.current) {

src/plugins/data_view_management/public/components/index_pattern_table/index_pattern_table.tsx

@@ -217,7 +217,7 @@ export const IndexPatternTable = ({
         <div>
           <EuiLink
             {...reactRouterNavigate(history, `patterns/${dataView.id}`)}
-            data-test-subj={`detail-link-${dataView.name}`}
+            data-test-subj={`detail-link-${dataView.getName()}`}
           >
             {dataView.getName()}
             {dataView.name ? (

src/plugins/data_views/common/constants.ts

@@ -58,6 +58,12 @@ export const PLUGIN_NAME = 'DataViews';
  */
 export const FIELDS_FOR_WILDCARD_PATH = '/internal/data_views/_fields_for_wildcard';
 
+/**
+ * Fields path. Like fields for wildcard but GET only
+ * @public
+ */
+export const FIELDS_PATH = '/internal/data_views/fields';
+
 /**
  * Existing indices path
  * @public

src/plugins/data_views/common/data_views/data_view.ts

@@ -61,6 +61,8 @@ export class DataView extends AbstractDataView implements DataViewBase {
    */
   public flattenHit: (hit: Record<string, unknown[]>, deep?: boolean) => Record<string, unknown>;
 
+  private etag: string | undefined;
+
   /**
    * constructor
    * @param config - config data and dependencies
@@ -77,6 +79,10 @@ export class DataView extends AbstractDataView implements DataViewBase {
     this.fields.replaceAll(Object.values(spec.fields || {}));
   }
 
+  getEtag = () => this.etag;
+
+  setEtag = (etag: string | undefined) => (this.etag = etag);
+
   /**
    * Returns scripted fields
    */

src/plugins/data_views/common/data_views/data_views.test.ts

@@ -158,10 +158,23 @@ describe('IndexPatterns', () => {
 
   test('force field refresh', async () => {
     const id = '1';
+    // make sure initial load and subsequent reloads use same params
+    const args = {
+      allowHidden: undefined,
+      allowNoIndex: true,
+      indexFilter: undefined,
+      metaFields: false,
+      pattern: 'something',
+      rollupIndex: undefined,
+      type: undefined,
+    };
+
     await indexPatterns.get(id);
     expect(apiClient.getFieldsForWildcard).toBeCalledTimes(1);
+    expect(apiClient.getFieldsForWildcard).toBeCalledWith(args);
     await indexPatterns.get(id, undefined, true);
     expect(apiClient.getFieldsForWildcard).toBeCalledTimes(2);
+    expect(apiClient.getFieldsForWildcard).toBeCalledWith(args);
   });
 
   test('getFieldsForWildcard called with allowNoIndex set to true as default ', async () => {