diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts index f4cbe2f03e966..8611e1ccd2cab 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts @@ -29,6 +29,7 @@ interface Config { mappingsString: string; pipelineString: string; pipelineId: string | null; + createPipeline: boolean; } export async function importData(props: Props, config: Config, setState: (state: unknown) => void) { @@ -41,6 +42,7 @@ export async function importData(props: Props, config: Config, setState: (state: mappingsString, pipelineString, pipelineId, + createPipeline, } = config; const { format } = results; @@ -86,7 +88,7 @@ export async function importData(props: Props, config: Config, setState: (state: let settings = {}; let mappings = {}; - let pipeline = {}; + let pipeline; try { settings = JSON.parse(indexSettingsString); @@ -109,7 +111,9 @@ export async function importData(props: Props, config: Config, setState: (state: } try { - pipeline = JSON.parse(pipelineString); + if (createPipeline) { + pipeline = JSON.parse(pipelineString) as IngestPipeline; + } } catch (error) { success = false; const parseError = i18n.translate('xpack.dataVisualizer.file.importView.parsePipelineError', { @@ -143,12 +147,7 @@ export async function importData(props: Props, config: Config, setState: (state: return; } - const initializeImportResp = await importer.initializeImport( - index, - settings, - mappings, - pipeline as IngestPipeline - ); + const initializeImportResp = await importer.initializeImport(index, settings, mappings, pipeline); const timeFieldName = importer.getTimeField(); setState({ timeFieldName }); @@ -158,14 +157,20 @@ export async function importData(props: Props, config: Config, setState: (state: indexCreatedStatus: getSuccess(indexCreated), }); - const pipelineCreated = initializeImportResp.pipelineId !== undefined; - if (indexCreated) { - setState({ - ingestPipelineCreatedStatus: pipelineCreated ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED, - pipelineId: pipelineCreated ? initializeImportResp.pipelineId : '', - }); + if (createPipeline) { + const pipelineCreated = initializeImportResp.pipelineId !== undefined; + if (indexCreated) { + setState({ + ingestPipelineCreatedStatus: pipelineCreated + ? IMPORT_STATUS.COMPLETE + : IMPORT_STATUS.FAILED, + pipelineId: pipelineCreated ? initializeImportResp.pipelineId : '', + }); + } + success = indexCreated && pipelineCreated; + } else { + success = indexCreated; } - success = indexCreated && pipelineCreated; if (success === false) { errors.push(initializeImportResp.error); diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js index aecf755c1c7c1..8481ed76e5654 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js @@ -109,6 +109,11 @@ export class ImportView extends Component { pipelineId, } = this.state; + const createPipeline = pipelineString !== ''; + this.setState({ + createPipeline, + }); + importData( { data, results, dataViewsContract, fileUpload }, { @@ -119,6 +124,7 @@ export class ImportView extends Component { mappingsString, pipelineString, pipelineId, + createPipeline, }, (state) => this.setState(state) ); diff --git a/x-pack/plugins/file_upload/public/importer/geo/abstract_geo_file_importer.tsx b/x-pack/plugins/file_upload/public/importer/geo/abstract_geo_file_importer.tsx index 2b366eef42b84..c79228bee933b 100644 --- a/x-pack/plugins/file_upload/public/importer/geo/abstract_geo_file_importer.tsx +++ b/x-pack/plugins/file_upload/public/importer/geo/abstract_geo_file_importer.tsx @@ -196,9 +196,12 @@ export class AbstractGeoFileImporter extends Importer implements GeoFileImporter data: chunks[i], settings: {}, mappings: {}, - ingestPipeline: { - id: pipelineId, - }, + ingestPipeline: + pipelineId !== undefined + ? { + id: pipelineId, + } + : undefined, }); if (!this._isActive) { diff --git a/x-pack/plugins/file_upload/public/importer/importer.ts b/x-pack/plugins/file_upload/public/importer/importer.ts index 6337e892868fe..6c556a3ef49ef 100644 --- a/x-pack/plugins/file_upload/public/importer/importer.ts +++ b/x-pack/plugins/file_upload/public/importer/importer.ts @@ -15,7 +15,13 @@ import { i18n } from '@kbn/i18n'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import { getHttp } from '../kibana_services'; import { MB } from '../../common/constants'; -import type { ImportDoc, ImportFailure, ImportResponse, IngestPipeline } from '../../common/types'; +import type { + ImportDoc, + ImportFailure, + ImportResponse, + IngestPipeline, + IngestPipelineWrapper, +} from '../../common/types'; import { CreateDocsResponse, IImporter, ImportResults } from './types'; const CHUNK_SIZE = 5000; @@ -79,26 +85,25 @@ export abstract class Importer implements IImporter { index: string, settings: IndicesIndexSettings, mappings: MappingTypeMapping, - pipeline: IngestPipeline + pipeline: IngestPipeline | undefined ) { - updatePipelineTimezone(pipeline); - - if (pipelineContainsSpecialProcessors(pipeline)) { - // pipeline contains processors which we know are slow - // so reduce the chunk size significantly to avoid timeouts - this._chunkSize = REDUCED_CHUNK_SIZE; + let ingestPipeline: IngestPipelineWrapper | undefined; + if (pipeline !== undefined) { + updatePipelineTimezone(pipeline); + + if (pipelineContainsSpecialProcessors(pipeline)) { + // pipeline contains processors which we know are slow + // so reduce the chunk size significantly to avoid timeouts + this._chunkSize = REDUCED_CHUNK_SIZE; + } + // if no pipeline has been supplied, + // send an empty object + ingestPipeline = { + id: `${index}-pipeline`, + pipeline, + }; } - // if no pipeline has been supplied, - // send an empty object - const ingestPipeline = - pipeline !== undefined - ? { - id: `${index}-pipeline`, - pipeline, - } - : {}; - this._index = index; this._pipeline = pipeline; @@ -139,9 +144,11 @@ export abstract class Importer implements IImporter { const chunks = createDocumentChunks(this._docArray, this._chunkSize); - const ingestPipeline = { - id: pipelineId, - }; + const ingestPipeline: IngestPipelineWrapper | undefined = pipelineId + ? { + id: pipelineId, + } + : undefined; let success = true; const failures: ImportFailure[] = []; @@ -345,10 +352,7 @@ export function callImportRoute({ data: ImportDoc[]; settings: IndicesIndexSettings; mappings: MappingTypeMapping; - ingestPipeline: { - id?: string; - pipeline?: IngestPipeline; - }; + ingestPipeline: IngestPipelineWrapper | undefined; }) { const query = id !== undefined ? { id } : {}; const body = JSON.stringify({ diff --git a/x-pack/plugins/file_upload/public/importer/types.ts b/x-pack/plugins/file_upload/public/importer/types.ts index 1199bb5a15c6a..4181e20528d00 100644 --- a/x-pack/plugins/file_upload/public/importer/types.ts +++ b/x-pack/plugins/file_upload/public/importer/types.ts @@ -43,7 +43,7 @@ export interface IImporter { index: string, settings: IndicesIndexSettings, mappings: MappingTypeMapping, - pipeline: IngestPipeline + pipeline: IngestPipeline | undefined ): Promise; import( id: string, diff --git a/x-pack/plugins/file_upload/server/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts index 2ae00ee1f9e08..afcec55107ff9 100644 --- a/x-pack/plugins/file_upload/server/import_data.ts +++ b/x-pack/plugins/file_upload/server/import_data.ts @@ -21,7 +21,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { index: string, settings: IndicesIndexSettings, mappings: MappingTypeMapping, - ingestPipeline: IngestPipelineWrapper, + ingestPipeline: IngestPipelineWrapper | undefined, data: InputData ): Promise { let createdIndex; @@ -29,7 +29,8 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { const docCount = data.length; try { - const { id: pipelineId, pipeline } = ingestPipeline; + const pipelineId = ingestPipeline?.id; + const pipeline = ingestPipeline?.pipeline; if (id === undefined) { // first chunk of data, create the index and id to return @@ -48,7 +49,6 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { createdPipelineId = pipelineId; } else { createdIndex = index; - createdPipelineId = pipelineId; } let failures: ImportFailure[] = []; @@ -109,7 +109,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { await asCurrentUser.indices.create({ index, body }, { maxRetries: 0 }); } - async function indexData(index: string, pipelineId: string, data: InputData) { + async function indexData(index: string, pipelineId: string | undefined, data: InputData) { try { const body = []; for (let i = 0; i < data.length; i++) { diff --git a/x-pack/plugins/file_upload/server/schemas.ts b/x-pack/plugins/file_upload/server/schemas.ts index b7250b45a82f9..94a4e9d9fa6f2 100644 --- a/x-pack/plugins/file_upload/server/schemas.ts +++ b/x-pack/plugins/file_upload/server/schemas.ts @@ -37,10 +37,12 @@ export const importFileBodySchema = schema.object({ /** Mappings */ mappings: schema.any(), /** Ingest pipeline definition */ - ingestPipeline: schema.object({ - id: schema.maybe(schema.string()), - pipeline: schema.maybe(schema.any()), - }), + ingestPipeline: schema.maybe( + schema.object({ + id: schema.maybe(schema.string()), + pipeline: schema.maybe(schema.any()), + }) + ), }); export const runtimeMappingsSchema = schema.object( diff --git a/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts b/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts index 3a859249fe234..9b4176d070c63 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts @@ -180,6 +180,42 @@ export default function ({ getService }: FtrProviderContext) { ingestedDocCount: 3, }, }, + { + suiteSuffix: 'with a file which does not generate a ingest pipeline', + filePath: require.resolve('./files_to_import/flights_small.json'), + indexName: 'user-import_4', + createIndexPattern: false, + fieldTypeFilters: [ML_JOB_FIELD_TYPES.KEYWORD], + fieldNameFilters: ['timestamp'], + expected: { + results: { + title: 'flights_small.json', + highlightedText: false, + }, + metricFields: [], + nonMetricFields: [ + { + fieldName: 'Carrier', + type: ML_JOB_FIELD_TYPES.KEYWORD, + docCountFormatted: '20 (100%)', + exampleCount: 4, + }, + { + fieldName: 'timestamp', + type: ML_JOB_FIELD_TYPES.KEYWORD, + docCountFormatted: '20 (100%)', + exampleCount: 11, + }, + ], + visibleMetricFieldsCount: 0, + totalMetricFieldsCount: 0, + populatedFieldsCount: 3, + totalFieldsCount: 25, + fieldTypeFiltersResultCount: 16, + fieldNameFiltersResultCount: 1, + ingestedDocCount: 20, + }, + }, ]; const testDataListNegative = [ diff --git a/x-pack/test/functional/apps/ml/data_visualizer/files_to_import/flights_small.json b/x-pack/test/functional/apps/ml/data_visualizer/files_to_import/flights_small.json new file mode 100644 index 0000000000000..32b5848f2d4ab --- /dev/null +++ b/x-pack/test/functional/apps/ml/data_visualizer/files_to_import/flights_small.json @@ -0,0 +1,20 @@ +{"FlightNum": "9HY9SWR", "DestCountry": "AU", "OriginWeather": "Sunny", "OriginCityName": "Frankfurt am Main", "AvgTicketPrice": 841.2656419677076, "DestWeather": "Rain", "Dest": "Sydney Kingsford Smith International Airport", "FlightDelayType": "No Delay", "OriginCountry": "DE", "dayOfWeek": 0, "DistanceKilometers": 16492.32665375846, "timestamp": "2018-01-01T00:00:00", "DestLocation": {"lat": "-33.94609833", "lon": "151.177002"}, "DestAirportID": "SYD", "Carrier": "Kibana Airlines", "Cancelled": false, "FlightTimeMin": 1030.7704158599038, "Origin": "Frankfurt am Main Airport", "OriginLocation": {"lat": "50.033333", "lon": "8.570556"}, "DestRegion": "SE-BD", "OriginAirportID": "FRA", "OriginRegion": "DE-HE", "DestCityName": "Sydney", "FlightTimeHour": 17.179506930998397, "FlightDelayMin": 0} +{"FlightNum": "X98CCZO", "DestCountry": "IT", "OriginWeather": "Clear", "OriginCityName": "Cape Town", "AvgTicketPrice": 882.9826615595518, "DestWeather": "Sunny", "Dest": "Venice Marco Polo Airport", "FlightDelayType": "No Delay", "OriginCountry": "ZA", "dayOfWeek": 0, "DistanceKilometers": 8823.40014044213, "timestamp": "2018-01-01T18:27:00", "DestLocation": {"lat": "45.505299", "lon": "12.3519"}, "DestAirportID": "VE05", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 464.3894810759016, "Origin": "Cape Town International Airport", "OriginLocation": {"lat": "-33.96480179", "lon": "18.60169983"}, "DestRegion": "IT-34", "OriginAirportID": "CPT", "OriginRegion": "SE-BD", "DestCityName": "Venice", "FlightTimeHour": 7.73982468459836, "FlightDelayMin": 0} +{"FlightNum": "UFK2WIZ", "DestCountry": "IT", "OriginWeather": "Rain", "OriginCityName": "Venice", "AvgTicketPrice": 190.6369038508356, "DestWeather": "Cloudy", "Dest": "Venice Marco Polo Airport", "FlightDelayType": "No Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 0.0, "timestamp": "2018-01-01T17:11:14", "DestLocation": {"lat": "45.505299", "lon": "12.3519"}, "DestAirportID": "VE05", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 0.0, "Origin": "Venice Marco Polo Airport", "OriginLocation": {"lat": "45.505299", "lon": "12.3519"}, "DestRegion": "IT-34", "OriginAirportID": "VE05", "OriginRegion": "IT-34", "DestCityName": "Venice", "FlightTimeHour": 0.0, "FlightDelayMin": 0} +{"FlightNum": "EAYQW69", "DestCountry": "IT", "OriginWeather": "Thunder & Lightning", "OriginCityName": "Naples", "AvgTicketPrice": 181.69421554118, "DestWeather": "Clear", "Dest": "Treviso-Sant'Angelo Airport", "FlightDelayType": "Weather Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 555.7377668725265, "timestamp": "2018-01-01", "DestLocation": {"lat": "45.648399", "lon": "12.1944"}, "DestAirportID": "TV01", "Carrier": "Kibana Airlines", "Cancelled": true, "FlightTimeMin": 222.74905899019436, "Origin": "Naples International Airport", "OriginLocation": {"lat": "40.886002", "lon": "14.2908"}, "DestRegion": "IT-34", "OriginAirportID": "NA01", "OriginRegion": "IT-72", "DestCityName": "Treviso", "FlightTimeHour": 3.712484316503239, "FlightDelayMin": 180} +{"FlightNum": "58U013N", "DestCountry": "CN", "OriginWeather": "Damaging Wind", "OriginCityName": "Mexico City", "AvgTicketPrice": 730.041778346198, "DestWeather": "Clear", "Dest": "Xi'an Xianyang International Airport", "FlightDelayType": "No Delay", "OriginCountry": "MX", "dayOfWeek": 0, "DistanceKilometers": 13358.24419986236, "timestamp": "2018-01-01T05:13:00", "DestLocation": {"lat": "34.447102", "lon": "108.751999"}, "DestAirportID": "XIY", "Carrier": "Kibana Airlines", "Cancelled": false, "FlightTimeMin": 785.7790705801389, "Origin": "Licenciado Benito Juarez International Airport", "OriginLocation": {"lat": "19.4363", "lon": "-99.072098"}, "DestRegion": "SE-BD", "OriginAirportID": "AICM", "OriginRegion": "MX-DIF", "DestCityName": "Xi'an", "FlightTimeHour": 13.096317843002314, "FlightDelayMin": 0} +{"FlightNum": "XEJ78I2", "DestCountry": "IT", "OriginWeather": "Rain", "OriginCityName": "Edmonton", "AvgTicketPrice": 418.1520890531832, "DestWeather": "Thunder & Lightning", "Dest": "Genoa Cristoforo Colombo Airport", "FlightDelayType": "No Delay", "OriginCountry": "CA", "dayOfWeek": 0, "DistanceKilometers": 7871.808813474433, "timestamp": "2018-01-01T01:43:03", "DestLocation": {"lat": "44.4133", "lon": "8.8375"}, "DestAirportID": "GE01", "Carrier": "JetBeats", "Cancelled": false, "FlightTimeMin": 393.5904406737217, "Origin": "Edmonton International Airport", "OriginLocation": {"lat": "53.30970001", "lon": "-113.5800018"}, "DestRegion": "IT-42", "OriginAirportID": "CYEG", "OriginRegion": "CA-AB", "DestCityName": "Genova", "FlightTimeHour": 6.5598406778953615, "FlightDelayMin": 0} +{"FlightNum": "EVARI8I", "DestCountry": "CH", "OriginWeather": "Clear", "OriginCityName": "Zurich", "AvgTicketPrice": 180.24681638061213, "DestWeather": "Hail", "Dest": "Zurich Airport", "FlightDelayType": "Security Delay", "OriginCountry": "CH", "dayOfWeek": 0, "DistanceKilometers": 0.0, "timestamp": "2018-01-01T13:49:53", "DestLocation": {"lat": "47.464699", "lon": "8.54917"}, "DestAirportID": "ZRH", "Carrier": "JetBeats", "Cancelled": false, "FlightTimeMin": 300.0, "Origin": "Zurich Airport", "OriginLocation": {"lat": "47.464699", "lon": "8.54917"}, "DestRegion": "CH-ZH", "OriginAirportID": "ZRH", "OriginRegion": "CH-ZH", "DestCityName": "Zurich", "FlightTimeHour": 5.0, "FlightDelayMin": 300} +{"FlightNum": "1IRBW25", "DestCountry": "CA", "OriginWeather": "Thunder & Lightning", "OriginCityName": "Rome", "AvgTicketPrice": 585.1843103083941, "DestWeather": "Clear", "Dest": "Ottawa Macdonald-Cartier International Airport", "FlightDelayType": "No Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 6764.367283910481, "timestamp": "2018-01-01T04:54:59", "DestLocation": {"lat": "45.32249832", "lon": "-75.66919708"}, "DestAirportID": "YOW", "Carrier": "Kibana Airlines", "Cancelled": false, "FlightTimeMin": 614.9424803554983, "Origin": "Ciampino___G. B. Pastine International Airport", "OriginLocation": {"lat": "41.7994", "lon": "12.5949"}, "DestRegion": "CA-ON", "OriginAirportID": "RM12", "OriginRegion": "IT-62", "DestCityName": "Ottawa", "FlightTimeHour": 10.249041339258305, "FlightDelayMin": 0} +{"FlightNum": "M05KE88", "DestCountry": "IN", "OriginWeather": "Heavy Fog", "OriginCityName": "Milan", "AvgTicketPrice": 960.8697358054351, "DestWeather": "Cloudy", "Dest": "Rajiv Gandhi International Airport", "FlightDelayType": "NAS Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 7044.367088850781, "timestamp": "2018-01-01T12:09:35", "DestLocation": {"lat": "17.23131752", "lon": "78.42985535"}, "DestAirportID": "HYD", "Carrier": "Kibana Airlines", "Cancelled": true, "FlightTimeMin": 602.0305907375651, "Origin": "Milano Linate Airport", "OriginLocation": {"lat": "45.445099", "lon": "9.27674"}, "DestRegion": "SE-BD", "OriginAirportID": "MI11", "OriginRegion": "IT-25", "DestCityName": "Hyderabad", "FlightTimeHour": 10.033843178959419, "FlightDelayMin": 15} +{"FlightNum": "SNI3M1Z", "DestCountry": "IT", "OriginWeather": "Cloudy", "OriginCityName": "Moscow", "AvgTicketPrice": 296.8777725965789, "DestWeather": "Rain", "Dest": "Treviso-Sant'Angelo Airport", "FlightDelayType": "No Delay", "OriginCountry": "RU", "dayOfWeek": 0, "DistanceKilometers": 2097.866595449369, "timestamp": "2018-01-01T12:09:35", "DestLocation": {"lat": "45.648399", "lon": "12.1944"}, "DestAirportID": "TV01", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 174.82221628744742, "Origin": "Sheremetyevo International Airport", "OriginLocation": {"lat": "55.972599", "lon": "37.4146"}, "DestRegion": "IT-34", "OriginAirportID": "SVO", "OriginRegion": "RU-MOS", "DestCityName": "Treviso", "FlightTimeHour": 2.9137036047907903, "FlightDelayMin": 0} +{"FlightNum": "JQ2XXQ5", "DestCountry": "FI", "OriginWeather": "Rain", "OriginCityName": "Albuquerque", "AvgTicketPrice": 906.4379477399872, "DestWeather": "Rain", "Dest": "Helsinki Vantaa Airport", "FlightDelayType": "No Delay", "OriginCountry": "US", "dayOfWeek": 0, "DistanceKilometers": 8551.76789268935, "timestamp": "2018-01-01T22:06:14", "DestLocation": {"lat": "60.31719971", "lon": "24.9633007"}, "DestAirportID": "HEL", "Carrier": "JetBeats", "Cancelled": false, "FlightTimeMin": 503.04517015819704, "Origin": "Albuquerque International Sunport Airport", "OriginLocation": {"lat": "35.040199", "lon": "-106.609001"}, "DestRegion": "FI-ES", "OriginAirportID": "ABQ", "OriginRegion": "US-NM", "DestCityName": "Helsinki", "FlightTimeHour": 8.384086169303284, "FlightDelayMin": 0} +{"FlightNum": "V30ITD0", "DestCountry": "AT", "OriginWeather": "Rain", "OriginCityName": "Venice", "AvgTicketPrice": 704.4637710312036, "DestWeather": "Cloudy", "Dest": "Vienna International Airport", "FlightDelayType": "No Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 432.90022115088834, "timestamp": "2018-01-01T11:52:34", "DestLocation": {"lat": "48.11029816", "lon": "16.56970024"}, "DestAirportID": "VIE", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 36.07501842924069, "Origin": "Venice Marco Polo Airport", "OriginLocation": {"lat": "45.505299", "lon": "12.3519"}, "DestRegion": "AT-9", "OriginAirportID": "VE05", "OriginRegion": "IT-34", "DestCityName": "Vienna", "FlightTimeHour": 0.6012503071540115, "FlightDelayMin": 0} +{"FlightNum": "P0WMFH7", "DestCountry": "CN", "OriginWeather": "Heavy Fog", "OriginCityName": "Mexico City", "AvgTicketPrice": 922.499077027416, "DestWeather": "Clear", "Dest": "Shanghai Pudong International Airport", "FlightDelayType": "No Delay", "OriginCountry": "MX", "dayOfWeek": 0, "DistanceKilometers": 12915.599427519877, "timestamp": "2018-01-01T02:13:46", "DestLocation": {"lat": "31.14340019", "lon": "121.8050003"}, "DestAirportID": "PVG", "Carrier": "Logstash Airways", "Cancelled": true, "FlightTimeMin": 679.7683909220988, "Origin": "Licenciado Benito Juarez International Airport", "OriginLocation": {"lat": "19.4363", "lon": "-99.072098"}, "DestRegion": "SE-BD", "OriginAirportID": "AICM", "OriginRegion": "MX-DIF", "DestCityName": "Shanghai", "FlightTimeHour": 11.32947318203498, "FlightDelayMin": 0} +{"FlightNum": "VT9O2KD", "DestCountry": "CA", "OriginWeather": "Rain", "OriginCityName": "Naples", "AvgTicketPrice": 374.9592762864519, "DestWeather": "Rain", "Dest": "Ottawa Macdonald-Cartier International Airport", "FlightDelayType": "No Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 6938.783925856956, "timestamp": "2018-01-01T14:21:13", "DestLocation": {"lat": "45.32249832", "lon": "-75.66919708"}, "DestAirportID": "YOW", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 330.41828218366453, "Origin": "Naples International Airport", "OriginLocation": {"lat": "40.886002", "lon": "14.2908"}, "DestRegion": "CA-ON", "OriginAirportID": "NA01", "OriginRegion": "IT-72", "DestCityName": "Ottawa", "FlightTimeHour": 5.506971369727742, "FlightDelayMin": 0} +{"FlightNum": "NRHSVG8", "DestCountry": "PR", "OriginWeather": "Cloudy", "OriginCityName": "Rome", "AvgTicketPrice": 552.9173708459598, "DestWeather": "Clear", "Dest": "Luis Munoz Marin International Airport", "FlightDelayType": "No Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 7735.755582005642, "timestamp": "2018-01-01T17:42:53", "DestLocation": {"lat": "18.43939972", "lon": "-66.00180054"}, "DestAirportID": "SJU", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 407.1450306318759, "Origin": "Ciampino___G. B. Pastine International Airport", "OriginLocation": {"lat": "41.7994", "lon": "12.5949"}, "DestRegion": "PR-U-A", "OriginAirportID": "RM12", "OriginRegion": "IT-62", "DestCityName": "San Juan", "FlightTimeHour": 6.7857505105312645, "FlightDelayMin": 0} +{"FlightNum": "YIPS2BZ", "DestCountry": "DE", "OriginWeather": "Thunder & Lightning", "OriginCityName": "Chengdu", "AvgTicketPrice": 566.4875569256166, "DestWeather": "Sunny", "Dest": "Cologne Bonn Airport", "FlightDelayType": "No Delay", "OriginCountry": "CN", "dayOfWeek": 0, "DistanceKilometers": 7880.551894165264, "timestamp": "2018-01-01T19:55:32", "DestLocation": {"lat": "50.86589813", "lon": "7.142739773"}, "DestAirportID": "CGN", "Carrier": "Kibana Airlines", "Cancelled": true, "FlightTimeMin": 656.7126578471053, "Origin": "Chengdu Shuangliu International Airport", "OriginLocation": {"lat": "30.57850075", "lon": "103.9469986"}, "DestRegion": "DE-NW", "OriginAirportID": "CTU", "OriginRegion": "SE-BD", "DestCityName": "Cologne", "FlightTimeHour": 10.945210964118422, "FlightDelayMin": 0} +{"FlightNum": "C7IBZ42", "DestCountry": "IT", "OriginWeather": "Thunder & Lightning", "OriginCityName": "Mexico City", "AvgTicketPrice": 989.9527866266118, "DestWeather": "Damaging Wind", "Dest": "Venice Marco Polo Airport", "FlightDelayType": "No Delay", "OriginCountry": "MX", "dayOfWeek": 0, "DistanceKilometers": 10049.394341914194, "timestamp": "2018-01-01T07:49:27", "DestLocation": {"lat": "45.505299", "lon": "12.3519"}, "DestAirportID": "VE05", "Carrier": "Logstash Airways", "Cancelled": true, "FlightTimeMin": 773.0303339933996, "Origin": "Licenciado Benito Juarez International Airport", "OriginLocation": {"lat": "19.4363", "lon": "-99.072098"}, "DestRegion": "IT-34", "OriginAirportID": "AICM", "OriginRegion": "MX-DIF", "DestCityName": "Venice", "FlightTimeHour": 12.883838899889993, "FlightDelayMin": 0} +{"FlightNum": "7TTZM4I", "DestCountry": "AR", "OriginWeather": "Rain", "OriginCityName": "Cleveland", "AvgTicketPrice": 569.6132552035547, "DestWeather": "Cloudy", "Dest": "Ministro Pistarini International Airport", "FlightDelayType": "NAS Delay", "OriginCountry": "US", "dayOfWeek": 0, "DistanceKilometers": 8771.31996172178, "timestamp": "2018-01-01T01:30:47", "DestLocation": {"lat": "-34.8222", "lon": "-58.5358"}, "DestAirportID": "EZE", "Carrier": "ES-Air", "Cancelled": false, "FlightTimeMin": 704.7169201324446, "Origin": "Cleveland Hopkins International Airport", "OriginLocation": {"lat": "41.4117012", "lon": "-81.84980011"}, "DestRegion": "SE-BD", "OriginAirportID": "CLE", "OriginRegion": "US-OH", "DestCityName": "Buenos Aires", "FlightTimeHour": 11.745282002207409, "FlightDelayMin": 30} +{"FlightNum": "CSW89S3", "DestCountry": "CN", "OriginWeather": "Hail", "OriginCityName": "Olenegorsk", "AvgTicketPrice": 277.4297073844173, "DestWeather": "Clear", "Dest": "Shanghai Pudong International Airport", "FlightDelayType": "No Delay", "OriginCountry": "RU", "dayOfWeek": 0, "DistanceKilometers": 6763.2019332738655, "timestamp": "2018-01-01T07:58:17", "DestLocation": {"lat": "31.14340019", "lon": "121.8050003"}, "DestAirportID": "PVG", "Carrier": "ES-Air", "Cancelled": false, "FlightTimeMin": 355.95799648809816, "Origin": "Olenya Air Base", "OriginLocation": {"lat": "68.15180206", "lon": "33.46390152"}, "DestRegion": "SE-BD", "OriginAirportID": "XLMO", "OriginRegion": "RU-MUR", "DestCityName": "Shanghai", "FlightTimeHour": 5.932633274801636, "FlightDelayMin": 0} +{"FlightNum": "RBFKZBX", "DestCountry": "IN", "OriginWeather": "Cloudy", "OriginCityName": "Casper", "AvgTicketPrice": 772.1008456460212, "DestWeather": "Clear", "Dest": "Indira Gandhi International Airport", "FlightDelayType": "Late Aircraft Delay", "OriginCountry": "US", "dayOfWeek": 0, "DistanceKilometers": 12081.834801603853, "timestamp": "2018-01-01T00:02:06", "DestLocation": {"lat": "28.5665", "lon": "77.103104"}, "DestAirportID": "DEL", "Carrier": "JetBeats", "Cancelled": false, "FlightTimeMin": 875.1146751002408, "Origin": "Casper-Natrona County International Airport", "OriginLocation": {"lat": "42.90800095", "lon": "-106.4639969"}, "DestRegion": "SE-BD", "OriginAirportID": "CPR", "OriginRegion": "US-WY", "DestCityName": "New Delhi", "FlightTimeHour": 14.585244585004013, "FlightDelayMin": 120}