[ML] Sharing file upload's message importer class (elastic#176047)

`MessageImporter` is now exported from the file upload plugin so it can
be used in a future a future PR to parse a log file.

Updates types to allow us to specify the type doc returned after
parsing.
Adds the ability to specify an optional limit on the number of lines
parsed.

x-pack/plugins/file_upload/common/types.ts

@@ -69,6 +69,7 @@ export interface FindFileStructureResponse {
   joda_timestamp_formats?: string[];
   timestamp_field?: string;
   should_trim_fields?: boolean;
+  ecs_compatibility?: string;
 }
 
 export interface FindFileStructureErrorResponse {

x-pack/plugins/file_upload/public/importer/geo/abstract_geo_file_importer.tsx

@@ -314,7 +314,7 @@ export class AbstractGeoFileImporter extends Importer implements GeoFileImporter
     throw new Error('read(data: ArrayBuffer) not supported, use previewFile and import instead.');
   }
 
-  protected _createDocs(text: string): CreateDocsResponse {
+  protected _createDocs(text: string): CreateDocsResponse<ImportDoc> {
     throw new Error('_createDocs not implemented.');
   }
 }

x-pack/plugins/file_upload/public/importer/importer.ts

@@ -73,7 +73,7 @@ export abstract class Importer implements IImporter {
     return { success: true };
   }
 
-  protected abstract _createDocs(t: string, isLastPart: boolean): CreateDocsResponse;
+  protected abstract _createDocs(t: string, isLastPart: boolean): CreateDocsResponse<ImportDoc>;
 
   public async initializeImport(
     index: string,

x-pack/plugins/file_upload/public/importer/index.ts

@@ -8,3 +8,4 @@
 export { importerFactory } from './importer_factory';
 export { validateFile } from './validate_file';
 export * from './types';
+export { MessageImporter } from './message_importer';

x-pack/plugins/file_upload/public/importer/message_importer.ts

@@ -6,8 +6,8 @@
  */
 
 import { Importer } from './importer';
-import { ImportDocMessage } from '../../common/types';
-import { CreateDocsResponse, ImportFactoryOptions } from './types';
+import type { ImportDocMessage } from '../../common/types';
+import type { CreateDocsResponse, ImportFactoryOptions } from './types';
 
 export class MessageImporter extends Importer {
   private _excludeLinesRegex: RegExp | null;
@@ -30,7 +30,11 @@ export class MessageImporter extends Importer {
   // multiline_start_pattern regex
   // if it does, it is a legitimate end of line and can be pushed into the list,
   // if not, it must be a newline char inside a field value, so keep looking.
-  protected _createDocs(text: string, isLastPart: boolean): CreateDocsResponse {
+  protected _createDocs(
+    text: string,
+    isLastPart: boolean,
+    lineLimit?: number
+  ): CreateDocsResponse<ImportDocMessage> {
     let remainder = 0;
     try {
       const docs: ImportDocMessage[] = [];
@@ -53,6 +57,10 @@ export class MessageImporter extends Importer {
         } else {
           line += char;
         }
+
+        if (lineLimit !== undefined && docs.length >= lineLimit) {
+          break;
+        }
       }
 
       remainder = line.length;

x-pack/plugins/file_upload/public/importer/ndjson_importer.ts

@@ -13,7 +13,7 @@ export class NdjsonImporter extends Importer {
     super();
   }
 
-  protected _createDocs(json: string, isLastPart: boolean): CreateDocsResponse {
+  protected _createDocs(json: string, isLastPart: boolean): CreateDocsResponse<string> {
     let remainder = 0;
     try {
       const splitJson = json.split(/}\s*\n/);

x-pack/plugins/file_upload/public/importer/types.ts

@@ -25,17 +25,16 @@ export interface ImportResults {
   error?: any;
 }
 
-export interface CreateDocsResponse {
+export interface CreateDocsResponse<T extends ImportDoc> {
   success: boolean;
   remainder: number;
-  docs: ImportDoc[];
+  docs: T[];
   error?: any;
 }
 
 export interface ImportFactoryOptions {
   excludeLinesPattern?: string;
   multilineStartPattern?: string;
-  importConfig: ImportConfig;
 }
 
 export interface IImporter {

x-pack/plugins/file_upload/public/index.ts

@@ -16,3 +16,4 @@ export type { Props as IndexNameFormProps } from './components/geo_upload_form/i
 export type { FileUploadPluginStart } from './plugin';
 export type { FileUploadComponentProps, FileUploadGeoResults } from './lazy_load_bundle';
 export type { IImporter } from './importer/types';
+export { MessageImporter } from './importer';