script field support, fix field formatters

x-pack/plugins/reporting/export_types/csv/server/lib/flatten_hit.js

@@ -6,6 +6,7 @@
 
 import _ from 'lodash';
 
+// FIXME shouldn't this really use a service provided by @kbn-es-query? The logic here can part from Discover
 export function createFlattenHit(fields, metaFields, conflictedTypesFields) {
   const flattenSource = (flat, obj, keyPrefix) => {
     keyPrefix = keyPrefix ? keyPrefix + '.' : '';

x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv_search.ts

@@ -35,7 +35,15 @@ interface CsvResultFromSearch {
 }
 
 type EndpointCaller = (method: string, params: any) => Promise<any>;
-type FormatsMap = Map<string, any>;
+type FormatsMap = Map<
+  string,
+  {
+    id: string;
+    params: {
+      pattern: string;
+    };
+  }
+>;
 
 interface GenerateCsvParams {
   searchRequest: SearchRequest;
@@ -103,6 +111,13 @@ const getTimebasedParts = (
   return { combinedFilter, includes, timezone };
 };
 
+interface IndexPatternField {
+  scripted: boolean;
+  lang?: string;
+  script?: string;
+  name: string;
+}
+
 export async function generateCsvSearch(
   req: Request,
   server: KbnServer,
@@ -132,6 +147,19 @@ export async function generateCsvSearch(
   const esQueryConfig = await getEsQueryConfig(uiConfig);
   const [sortField, sortOrder] = savedSearchObjectAttr.sort;
 
+  const scriptFields = indexPatternSavedObject.fields
+    .filter((f: IndexPatternField) => f.scripted === true)
+    .reduce((accum: any, curr: IndexPatternField) => {
+      return {
+        ...accum,
+        [curr.name]: {
+          script: {
+            source: curr.script,
+            lang: curr.lang,
+          },
+        },
+      };
+    }, {});
   const searchRequest: SearchRequest = {
     index: indexPatternSavedObject.title,
     body: {
@@ -143,7 +171,7 @@ export async function generateCsvSearch(
         combinedFilter,
         esQueryConfig
       ),
-      script_fields: {},
+      script_fields: scriptFields,
       sort: [{ [sortField]: { order: sortOrder } }],
     },
   };
@@ -152,9 +180,9 @@ export async function generateCsvSearch(
   const callCluster = (...params: any[]) => callWithRequest(req, ...params);
   const config = server.config();
   const [formatsMap, uiSettings] = await Promise.all([
-    fieldFormatServiceFactory(uiConfig).then((fieldFormats: any) =>
-      fieldFormatMapFactory(indexPatternSavedObject, fieldFormats)
-    ),
+    fieldFormatServiceFactory(uiConfig).then((fieldFormats: any) => {
+      return fieldFormatMapFactory(indexPatternSavedObject, fieldFormats);
+    }),
     getUiSettings(uiConfig),
   ]);
 

x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/get_data_source.ts

@@ -42,19 +42,16 @@ export async function getDataSource(
     }
   }
   try {
-    const { attributes: indexPatternAttributes } = await savedObjectsClient.get(
-      'index-pattern',
-      indexPatternId
-    );
-    let fields = '[]';
-    let fieldFormatMap = '{}';
-    if (indexPatternAttributes.fields) {
-      fields = JSON.parse(indexPatternAttributes.fields);
-    }
-    if (indexPatternAttributes.fieldFormatMap) {
-      fieldFormatMap = JSON.parse(indexPatternAttributes.fieldFormatMap);
-    }
-    indexPatternSavedObject = { ...indexPatternAttributes, fields, fieldFormatMap };
+    const { attributes } = await savedObjectsClient.get('index-pattern', indexPatternId);
+    const { fields, title, timeFieldName } = attributes;
+    const parsedFields = fields ? JSON.parse(fields) : [];
+
+    indexPatternSavedObject = {
+      fields: parsedFields,
+      title,
+      timeFieldName,
+      attributes,
+    };
   } catch (err) {
     throw new Error(`Could not get index pattern saved object! ${err}`);
   }

x-pack/plugins/reporting/export_types/csv_from_savedobject/types.d.ts

@@ -76,12 +76,18 @@ export interface SavedObject {
   references: SavedObjectReference[];
 }
 
+/* This object is passed to different helpers in different parts of the code
+   - packages/kbn-es-query/src/es_query/build_es_query
+   - x-pack/plugins/reporting/export_types/csv/server/lib/field_format_map
+   The structure has redundant parts and json-parsed / json-unparsed versions of the same data
+ */
 export interface IndexPatternSavedObject {
   title: string;
   timeFieldName: string;
   fields: any[];
-  fieldFormatMap: {
-    [key: string]: { id: string; params: { pattern: string } };
+  attributes: {
+    fieldFormatMap: string;
+    fields: string;
   };
 }
 

x-pack/test/api_integration/config.js

@@ -12,8 +12,7 @@ import {
   InfraOpsGraphQLProvider
 } from './services';
 
-export default async function ({ readConfigFile }) {
-
+export async function getApiIntegrationConfig({ readConfigFile }) {
   const kibanaAPITestsConfig = await readConfigFile(require.resolve('../../../test/api_integration/config.js'));
   const xPackFunctionalTestsConfig = await readConfigFile(require.resolve('../functional/config.js'));
   const kibanaCommonConfig = await readConfigFile(require.resolve('../../../test/common/config.js'));
@@ -47,3 +46,5 @@ export default async function ({ readConfigFile }) {
     esTestCluster: xPackFunctionalTestsConfig.get('esTestCluster'),
   };
 }
+
+export default getApiIntegrationConfig;

x-pack/test/functional/es_archives/reporting/sales/mappings.json

@@ -304,8 +304,6 @@
 {
   "type": "index",
   "value": {
-    "aliases": {
-    },
     "index": "sales",
     "mappings": {
       "properties": {