From 1291765832986d3fec3a62bd4858575c9752e940 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 25 Sep 2024 15:30:30 +0100 Subject: [PATCH] [ML] Fix file upload with no ingest pipeline (#193744) With some datasets the find structure api will not generate an ingest pipeline. A recent [change](https://github.com/elastic/kibana/pull/186956) to how we catch and display errors during file upload means an upload with no pipeline now produces an error which aborts the upload. Previously all pipeline creation errors were ignored and hidden from the user. This PR changes changes the file upload endpoint to allow it to receive no ingest pipeline and also changes the UI to not display the pipeline creation step during upload. This file can be used to test the fix. https://github.com/elastic/eland/blob/main/tests/flights.json.gz (cherry picked from commit ee1a147baca52dca5703663d35b66e7c44f3b676) --- .../components/import_view/import.ts | 35 ++++++------ .../components/import_view/import_view.js | 6 +++ .../geo/abstract_geo_file_importer.tsx | 9 ++-- .../file_upload/public/importer/importer.ts | 54 ++++++++++--------- .../file_upload/public/importer/types.ts | 2 +- .../plugins/file_upload/server/import_data.ts | 8 +-- x-pack/plugins/file_upload/server/schemas.ts | 10 ++-- .../data_visualizer/file_data_visualizer.ts | 36 +++++++++++++ .../files_to_import/flights_small.json | 20 +++++++ 9 files changed, 128 insertions(+), 52 deletions(-) create mode 100644 x-pack/test/functional/apps/ml/data_visualizer/files_to_import/flights_small.json diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts index f4cbe2f03e966..8611e1ccd2cab 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import.ts @@ -29,6 +29,7 @@ interface Config { mappingsString: string; pipelineString: string; pipelineId: string | null; + createPipeline: boolean; } export async function importData(props: Props, config: Config, setState: (state: unknown) => void) { @@ -41,6 +42,7 @@ export async function importData(props: Props, config: Config, setState: (state: mappingsString, pipelineString, pipelineId, + createPipeline, } = config; const { format } = results; @@ -86,7 +88,7 @@ export async function importData(props: Props, config: Config, setState: (state: let settings = {}; let mappings = {}; - let pipeline = {}; + let pipeline; try { settings = JSON.parse(indexSettingsString); @@ -109,7 +111,9 @@ export async function importData(props: Props, config: Config, setState: (state: } try { - pipeline = JSON.parse(pipelineString); + if (createPipeline) { + pipeline = JSON.parse(pipelineString) as IngestPipeline; + } } catch (error) { success = false; const parseError = i18n.translate('xpack.dataVisualizer.file.importView.parsePipelineError', { @@ -143,12 +147,7 @@ export async function importData(props: Props, config: Config, setState: (state: return; } - const initializeImportResp = await importer.initializeImport( - index, - settings, - mappings, - pipeline as IngestPipeline - ); + const initializeImportResp = await importer.initializeImport(index, settings, mappings, pipeline); const timeFieldName = importer.getTimeField(); setState({ timeFieldName }); @@ -158,14 +157,20 @@ export async function importData(props: Props, config: Config, setState: (state: indexCreatedStatus: getSuccess(indexCreated), }); - const pipelineCreated = initializeImportResp.pipelineId !== undefined; - if (indexCreated) { - setState({ - ingestPipelineCreatedStatus: pipelineCreated ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED, - pipelineId: pipelineCreated ? initializeImportResp.pipelineId : '', - }); + if (createPipeline) { + const pipelineCreated = initializeImportResp.pipelineId !== undefined; + if (indexCreated) { + setState({ + ingestPipelineCreatedStatus: pipelineCreated + ? IMPORT_STATUS.COMPLETE + : IMPORT_STATUS.FAILED, + pipelineId: pipelineCreated ? initializeImportResp.pipelineId : '', + }); + } + success = indexCreated && pipelineCreated; + } else { + success = indexCreated; } - success = indexCreated && pipelineCreated; if (success === false) { errors.push(initializeImportResp.error); diff --git a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js index aecf755c1c7c1..8481ed76e5654 100644 --- a/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js +++ b/x-pack/plugins/data_visualizer/public/application/file_data_visualizer/components/import_view/import_view.js @@ -109,6 +109,11 @@ export class ImportView extends Component { pipelineId, } = this.state; + const createPipeline = pipelineString !== ''; + this.setState({ + createPipeline, + }); + importData( { data, results, dataViewsContract, fileUpload }, { @@ -119,6 +124,7 @@ export class ImportView extends Component { mappingsString, pipelineString, pipelineId, + createPipeline, }, (state) => this.setState(state) ); diff --git a/x-pack/plugins/file_upload/public/importer/geo/abstract_geo_file_importer.tsx b/x-pack/plugins/file_upload/public/importer/geo/abstract_geo_file_importer.tsx index 2b366eef42b84..c79228bee933b 100644 --- a/x-pack/plugins/file_upload/public/importer/geo/abstract_geo_file_importer.tsx +++ b/x-pack/plugins/file_upload/public/importer/geo/abstract_geo_file_importer.tsx @@ -196,9 +196,12 @@ export class AbstractGeoFileImporter extends Importer implements GeoFileImporter data: chunks[i], settings: {}, mappings: {}, - ingestPipeline: { - id: pipelineId, - }, + ingestPipeline: + pipelineId !== undefined + ? { + id: pipelineId, + } + : undefined, }); if (!this._isActive) { diff --git a/x-pack/plugins/file_upload/public/importer/importer.ts b/x-pack/plugins/file_upload/public/importer/importer.ts index 6337e892868fe..6c556a3ef49ef 100644 --- a/x-pack/plugins/file_upload/public/importer/importer.ts +++ b/x-pack/plugins/file_upload/public/importer/importer.ts @@ -15,7 +15,13 @@ import { i18n } from '@kbn/i18n'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import { getHttp } from '../kibana_services'; import { MB } from '../../common/constants'; -import type { ImportDoc, ImportFailure, ImportResponse, IngestPipeline } from '../../common/types'; +import type { + ImportDoc, + ImportFailure, + ImportResponse, + IngestPipeline, + IngestPipelineWrapper, +} from '../../common/types'; import { CreateDocsResponse, IImporter, ImportResults } from './types'; const CHUNK_SIZE = 5000; @@ -79,26 +85,25 @@ export abstract class Importer implements IImporter { index: string, settings: IndicesIndexSettings, mappings: MappingTypeMapping, - pipeline: IngestPipeline + pipeline: IngestPipeline | undefined ) { - updatePipelineTimezone(pipeline); - - if (pipelineContainsSpecialProcessors(pipeline)) { - // pipeline contains processors which we know are slow - // so reduce the chunk size significantly to avoid timeouts - this._chunkSize = REDUCED_CHUNK_SIZE; + let ingestPipeline: IngestPipelineWrapper | undefined; + if (pipeline !== undefined) { + updatePipelineTimezone(pipeline); + + if (pipelineContainsSpecialProcessors(pipeline)) { + // pipeline contains processors which we know are slow + // so reduce the chunk size significantly to avoid timeouts + this._chunkSize = REDUCED_CHUNK_SIZE; + } + // if no pipeline has been supplied, + // send an empty object + ingestPipeline = { + id: `${index}-pipeline`, + pipeline, + }; } - // if no pipeline has been supplied, - // send an empty object - const ingestPipeline = - pipeline !== undefined - ? { - id: `${index}-pipeline`, - pipeline, - } - : {}; - this._index = index; this._pipeline = pipeline; @@ -139,9 +144,11 @@ export abstract class Importer implements IImporter { const chunks = createDocumentChunks(this._docArray, this._chunkSize); - const ingestPipeline = { - id: pipelineId, - }; + const ingestPipeline: IngestPipelineWrapper | undefined = pipelineId + ? { + id: pipelineId, + } + : undefined; let success = true; const failures: ImportFailure[] = []; @@ -345,10 +352,7 @@ export function callImportRoute({ data: ImportDoc[]; settings: IndicesIndexSettings; mappings: MappingTypeMapping; - ingestPipeline: { - id?: string; - pipeline?: IngestPipeline; - }; + ingestPipeline: IngestPipelineWrapper | undefined; }) { const query = id !== undefined ? { id } : {}; const body = JSON.stringify({ diff --git a/x-pack/plugins/file_upload/public/importer/types.ts b/x-pack/plugins/file_upload/public/importer/types.ts index 1199bb5a15c6a..4181e20528d00 100644 --- a/x-pack/plugins/file_upload/public/importer/types.ts +++ b/x-pack/plugins/file_upload/public/importer/types.ts @@ -43,7 +43,7 @@ export interface IImporter { index: string, settings: IndicesIndexSettings, mappings: MappingTypeMapping, - pipeline: IngestPipeline + pipeline: IngestPipeline | undefined ): Promise; import( id: string, diff --git a/x-pack/plugins/file_upload/server/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts index 2ae00ee1f9e08..afcec55107ff9 100644 --- a/x-pack/plugins/file_upload/server/import_data.ts +++ b/x-pack/plugins/file_upload/server/import_data.ts @@ -21,7 +21,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { index: string, settings: IndicesIndexSettings, mappings: MappingTypeMapping, - ingestPipeline: IngestPipelineWrapper, + ingestPipeline: IngestPipelineWrapper | undefined, data: InputData ): Promise { let createdIndex; @@ -29,7 +29,8 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { const docCount = data.length; try { - const { id: pipelineId, pipeline } = ingestPipeline; + const pipelineId = ingestPipeline?.id; + const pipeline = ingestPipeline?.pipeline; if (id === undefined) { // first chunk of data, create the index and id to return @@ -48,7 +49,6 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { createdPipelineId = pipelineId; } else { createdIndex = index; - createdPipelineId = pipelineId; } let failures: ImportFailure[] = []; @@ -109,7 +109,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { await asCurrentUser.indices.create({ index, body }, { maxRetries: 0 }); } - async function indexData(index: string, pipelineId: string, data: InputData) { + async function indexData(index: string, pipelineId: string | undefined, data: InputData) { try { const body = []; for (let i = 0; i < data.length; i++) { diff --git a/x-pack/plugins/file_upload/server/schemas.ts b/x-pack/plugins/file_upload/server/schemas.ts index b7250b45a82f9..94a4e9d9fa6f2 100644 --- a/x-pack/plugins/file_upload/server/schemas.ts +++ b/x-pack/plugins/file_upload/server/schemas.ts @@ -37,10 +37,12 @@ export const importFileBodySchema = schema.object({ /** Mappings */ mappings: schema.any(), /** Ingest pipeline definition */ - ingestPipeline: schema.object({ - id: schema.maybe(schema.string()), - pipeline: schema.maybe(schema.any()), - }), + ingestPipeline: schema.maybe( + schema.object({ + id: schema.maybe(schema.string()), + pipeline: schema.maybe(schema.any()), + }) + ), }); export const runtimeMappingsSchema = schema.object( diff --git a/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts b/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts index 3a859249fe234..9b4176d070c63 100644 --- a/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts +++ b/x-pack/test/functional/apps/ml/data_visualizer/file_data_visualizer.ts @@ -180,6 +180,42 @@ export default function ({ getService }: FtrProviderContext) { ingestedDocCount: 3, }, }, + { + suiteSuffix: 'with a file which does not generate a ingest pipeline', + filePath: require.resolve('./files_to_import/flights_small.json'), + indexName: 'user-import_4', + createIndexPattern: false, + fieldTypeFilters: [ML_JOB_FIELD_TYPES.KEYWORD], + fieldNameFilters: ['timestamp'], + expected: { + results: { + title: 'flights_small.json', + highlightedText: false, + }, + metricFields: [], + nonMetricFields: [ + { + fieldName: 'Carrier', + type: ML_JOB_FIELD_TYPES.KEYWORD, + docCountFormatted: '20 (100%)', + exampleCount: 4, + }, + { + fieldName: 'timestamp', + type: ML_JOB_FIELD_TYPES.KEYWORD, + docCountFormatted: '20 (100%)', + exampleCount: 11, + }, + ], + visibleMetricFieldsCount: 0, + totalMetricFieldsCount: 0, + populatedFieldsCount: 3, + totalFieldsCount: 25, + fieldTypeFiltersResultCount: 16, + fieldNameFiltersResultCount: 1, + ingestedDocCount: 20, + }, + }, ]; const testDataListNegative = [ diff --git a/x-pack/test/functional/apps/ml/data_visualizer/files_to_import/flights_small.json b/x-pack/test/functional/apps/ml/data_visualizer/files_to_import/flights_small.json new file mode 100644 index 0000000000000..32b5848f2d4ab --- /dev/null +++ b/x-pack/test/functional/apps/ml/data_visualizer/files_to_import/flights_small.json @@ -0,0 +1,20 @@ +{"FlightNum": "9HY9SWR", "DestCountry": "AU", "OriginWeather": "Sunny", "OriginCityName": "Frankfurt am Main", "AvgTicketPrice": 841.2656419677076, "DestWeather": "Rain", "Dest": "Sydney Kingsford Smith International Airport", "FlightDelayType": "No Delay", "OriginCountry": "DE", "dayOfWeek": 0, "DistanceKilometers": 16492.32665375846, "timestamp": "2018-01-01T00:00:00", "DestLocation": {"lat": "-33.94609833", "lon": "151.177002"}, "DestAirportID": "SYD", "Carrier": "Kibana Airlines", "Cancelled": false, "FlightTimeMin": 1030.7704158599038, "Origin": "Frankfurt am Main Airport", "OriginLocation": {"lat": "50.033333", "lon": "8.570556"}, "DestRegion": "SE-BD", "OriginAirportID": "FRA", "OriginRegion": "DE-HE", "DestCityName": "Sydney", "FlightTimeHour": 17.179506930998397, "FlightDelayMin": 0} +{"FlightNum": "X98CCZO", "DestCountry": "IT", "OriginWeather": "Clear", "OriginCityName": "Cape Town", "AvgTicketPrice": 882.9826615595518, "DestWeather": "Sunny", "Dest": "Venice Marco Polo Airport", "FlightDelayType": "No Delay", "OriginCountry": "ZA", "dayOfWeek": 0, "DistanceKilometers": 8823.40014044213, "timestamp": "2018-01-01T18:27:00", "DestLocation": {"lat": "45.505299", "lon": "12.3519"}, "DestAirportID": "VE05", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 464.3894810759016, "Origin": "Cape Town International Airport", "OriginLocation": {"lat": "-33.96480179", "lon": "18.60169983"}, "DestRegion": "IT-34", "OriginAirportID": "CPT", "OriginRegion": "SE-BD", "DestCityName": "Venice", "FlightTimeHour": 7.73982468459836, "FlightDelayMin": 0} +{"FlightNum": "UFK2WIZ", "DestCountry": "IT", "OriginWeather": "Rain", "OriginCityName": "Venice", "AvgTicketPrice": 190.6369038508356, "DestWeather": "Cloudy", "Dest": "Venice Marco Polo Airport", "FlightDelayType": "No Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 0.0, "timestamp": "2018-01-01T17:11:14", "DestLocation": {"lat": "45.505299", "lon": "12.3519"}, "DestAirportID": "VE05", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 0.0, "Origin": "Venice Marco Polo Airport", "OriginLocation": {"lat": "45.505299", "lon": "12.3519"}, "DestRegion": "IT-34", "OriginAirportID": "VE05", "OriginRegion": "IT-34", "DestCityName": "Venice", "FlightTimeHour": 0.0, "FlightDelayMin": 0} +{"FlightNum": "EAYQW69", "DestCountry": "IT", "OriginWeather": "Thunder & Lightning", "OriginCityName": "Naples", "AvgTicketPrice": 181.69421554118, "DestWeather": "Clear", "Dest": "Treviso-Sant'Angelo Airport", "FlightDelayType": "Weather Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 555.7377668725265, "timestamp": "2018-01-01", "DestLocation": {"lat": "45.648399", "lon": "12.1944"}, "DestAirportID": "TV01", "Carrier": "Kibana Airlines", "Cancelled": true, "FlightTimeMin": 222.74905899019436, "Origin": "Naples International Airport", "OriginLocation": {"lat": "40.886002", "lon": "14.2908"}, "DestRegion": "IT-34", "OriginAirportID": "NA01", "OriginRegion": "IT-72", "DestCityName": "Treviso", "FlightTimeHour": 3.712484316503239, "FlightDelayMin": 180} +{"FlightNum": "58U013N", "DestCountry": "CN", "OriginWeather": "Damaging Wind", "OriginCityName": "Mexico City", "AvgTicketPrice": 730.041778346198, "DestWeather": "Clear", "Dest": "Xi'an Xianyang International Airport", "FlightDelayType": "No Delay", "OriginCountry": "MX", "dayOfWeek": 0, "DistanceKilometers": 13358.24419986236, "timestamp": "2018-01-01T05:13:00", "DestLocation": {"lat": "34.447102", "lon": "108.751999"}, "DestAirportID": "XIY", "Carrier": "Kibana Airlines", "Cancelled": false, "FlightTimeMin": 785.7790705801389, "Origin": "Licenciado Benito Juarez International Airport", "OriginLocation": {"lat": "19.4363", "lon": "-99.072098"}, "DestRegion": "SE-BD", "OriginAirportID": "AICM", "OriginRegion": "MX-DIF", "DestCityName": "Xi'an", "FlightTimeHour": 13.096317843002314, "FlightDelayMin": 0} +{"FlightNum": "XEJ78I2", "DestCountry": "IT", "OriginWeather": "Rain", "OriginCityName": "Edmonton", "AvgTicketPrice": 418.1520890531832, "DestWeather": "Thunder & Lightning", "Dest": "Genoa Cristoforo Colombo Airport", "FlightDelayType": "No Delay", "OriginCountry": "CA", "dayOfWeek": 0, "DistanceKilometers": 7871.808813474433, "timestamp": "2018-01-01T01:43:03", "DestLocation": {"lat": "44.4133", "lon": "8.8375"}, "DestAirportID": "GE01", "Carrier": "JetBeats", "Cancelled": false, "FlightTimeMin": 393.5904406737217, "Origin": "Edmonton International Airport", "OriginLocation": {"lat": "53.30970001", "lon": "-113.5800018"}, "DestRegion": "IT-42", "OriginAirportID": "CYEG", "OriginRegion": "CA-AB", "DestCityName": "Genova", "FlightTimeHour": 6.5598406778953615, "FlightDelayMin": 0} +{"FlightNum": "EVARI8I", "DestCountry": "CH", "OriginWeather": "Clear", "OriginCityName": "Zurich", "AvgTicketPrice": 180.24681638061213, "DestWeather": "Hail", "Dest": "Zurich Airport", "FlightDelayType": "Security Delay", "OriginCountry": "CH", "dayOfWeek": 0, "DistanceKilometers": 0.0, "timestamp": "2018-01-01T13:49:53", "DestLocation": {"lat": "47.464699", "lon": "8.54917"}, "DestAirportID": "ZRH", "Carrier": "JetBeats", "Cancelled": false, "FlightTimeMin": 300.0, "Origin": "Zurich Airport", "OriginLocation": {"lat": "47.464699", "lon": "8.54917"}, "DestRegion": "CH-ZH", "OriginAirportID": "ZRH", "OriginRegion": "CH-ZH", "DestCityName": "Zurich", "FlightTimeHour": 5.0, "FlightDelayMin": 300} +{"FlightNum": "1IRBW25", "DestCountry": "CA", "OriginWeather": "Thunder & Lightning", "OriginCityName": "Rome", "AvgTicketPrice": 585.1843103083941, "DestWeather": "Clear", "Dest": "Ottawa Macdonald-Cartier International Airport", "FlightDelayType": "No Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 6764.367283910481, "timestamp": "2018-01-01T04:54:59", "DestLocation": {"lat": "45.32249832", "lon": "-75.66919708"}, "DestAirportID": "YOW", "Carrier": "Kibana Airlines", "Cancelled": false, "FlightTimeMin": 614.9424803554983, "Origin": "Ciampino___G. B. Pastine International Airport", "OriginLocation": {"lat": "41.7994", "lon": "12.5949"}, "DestRegion": "CA-ON", "OriginAirportID": "RM12", "OriginRegion": "IT-62", "DestCityName": "Ottawa", "FlightTimeHour": 10.249041339258305, "FlightDelayMin": 0} +{"FlightNum": "M05KE88", "DestCountry": "IN", "OriginWeather": "Heavy Fog", "OriginCityName": "Milan", "AvgTicketPrice": 960.8697358054351, "DestWeather": "Cloudy", "Dest": "Rajiv Gandhi International Airport", "FlightDelayType": "NAS Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 7044.367088850781, "timestamp": "2018-01-01T12:09:35", "DestLocation": {"lat": "17.23131752", "lon": "78.42985535"}, "DestAirportID": "HYD", "Carrier": "Kibana Airlines", "Cancelled": true, "FlightTimeMin": 602.0305907375651, "Origin": "Milano Linate Airport", "OriginLocation": {"lat": "45.445099", "lon": "9.27674"}, "DestRegion": "SE-BD", "OriginAirportID": "MI11", "OriginRegion": "IT-25", "DestCityName": "Hyderabad", "FlightTimeHour": 10.033843178959419, "FlightDelayMin": 15} +{"FlightNum": "SNI3M1Z", "DestCountry": "IT", "OriginWeather": "Cloudy", "OriginCityName": "Moscow", "AvgTicketPrice": 296.8777725965789, "DestWeather": "Rain", "Dest": "Treviso-Sant'Angelo Airport", "FlightDelayType": "No Delay", "OriginCountry": "RU", "dayOfWeek": 0, "DistanceKilometers": 2097.866595449369, "timestamp": "2018-01-01T12:09:35", "DestLocation": {"lat": "45.648399", "lon": "12.1944"}, "DestAirportID": "TV01", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 174.82221628744742, "Origin": "Sheremetyevo International Airport", "OriginLocation": {"lat": "55.972599", "lon": "37.4146"}, "DestRegion": "IT-34", "OriginAirportID": "SVO", "OriginRegion": "RU-MOS", "DestCityName": "Treviso", "FlightTimeHour": 2.9137036047907903, "FlightDelayMin": 0} +{"FlightNum": "JQ2XXQ5", "DestCountry": "FI", "OriginWeather": "Rain", "OriginCityName": "Albuquerque", "AvgTicketPrice": 906.4379477399872, "DestWeather": "Rain", "Dest": "Helsinki Vantaa Airport", "FlightDelayType": "No Delay", "OriginCountry": "US", "dayOfWeek": 0, "DistanceKilometers": 8551.76789268935, "timestamp": "2018-01-01T22:06:14", "DestLocation": {"lat": "60.31719971", "lon": "24.9633007"}, "DestAirportID": "HEL", "Carrier": "JetBeats", "Cancelled": false, "FlightTimeMin": 503.04517015819704, "Origin": "Albuquerque International Sunport Airport", "OriginLocation": {"lat": "35.040199", "lon": "-106.609001"}, "DestRegion": "FI-ES", "OriginAirportID": "ABQ", "OriginRegion": "US-NM", "DestCityName": "Helsinki", "FlightTimeHour": 8.384086169303284, "FlightDelayMin": 0} +{"FlightNum": "V30ITD0", "DestCountry": "AT", "OriginWeather": "Rain", "OriginCityName": "Venice", "AvgTicketPrice": 704.4637710312036, "DestWeather": "Cloudy", "Dest": "Vienna International Airport", "FlightDelayType": "No Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 432.90022115088834, "timestamp": "2018-01-01T11:52:34", "DestLocation": {"lat": "48.11029816", "lon": "16.56970024"}, "DestAirportID": "VIE", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 36.07501842924069, "Origin": "Venice Marco Polo Airport", "OriginLocation": {"lat": "45.505299", "lon": "12.3519"}, "DestRegion": "AT-9", "OriginAirportID": "VE05", "OriginRegion": "IT-34", "DestCityName": "Vienna", "FlightTimeHour": 0.6012503071540115, "FlightDelayMin": 0} +{"FlightNum": "P0WMFH7", "DestCountry": "CN", "OriginWeather": "Heavy Fog", "OriginCityName": "Mexico City", "AvgTicketPrice": 922.499077027416, "DestWeather": "Clear", "Dest": "Shanghai Pudong International Airport", "FlightDelayType": "No Delay", "OriginCountry": "MX", "dayOfWeek": 0, "DistanceKilometers": 12915.599427519877, "timestamp": "2018-01-01T02:13:46", "DestLocation": {"lat": "31.14340019", "lon": "121.8050003"}, "DestAirportID": "PVG", "Carrier": "Logstash Airways", "Cancelled": true, "FlightTimeMin": 679.7683909220988, "Origin": "Licenciado Benito Juarez International Airport", "OriginLocation": {"lat": "19.4363", "lon": "-99.072098"}, "DestRegion": "SE-BD", "OriginAirportID": "AICM", "OriginRegion": "MX-DIF", "DestCityName": "Shanghai", "FlightTimeHour": 11.32947318203498, "FlightDelayMin": 0} +{"FlightNum": "VT9O2KD", "DestCountry": "CA", "OriginWeather": "Rain", "OriginCityName": "Naples", "AvgTicketPrice": 374.9592762864519, "DestWeather": "Rain", "Dest": "Ottawa Macdonald-Cartier International Airport", "FlightDelayType": "No Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 6938.783925856956, "timestamp": "2018-01-01T14:21:13", "DestLocation": {"lat": "45.32249832", "lon": "-75.66919708"}, "DestAirportID": "YOW", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 330.41828218366453, "Origin": "Naples International Airport", "OriginLocation": {"lat": "40.886002", "lon": "14.2908"}, "DestRegion": "CA-ON", "OriginAirportID": "NA01", "OriginRegion": "IT-72", "DestCityName": "Ottawa", "FlightTimeHour": 5.506971369727742, "FlightDelayMin": 0} +{"FlightNum": "NRHSVG8", "DestCountry": "PR", "OriginWeather": "Cloudy", "OriginCityName": "Rome", "AvgTicketPrice": 552.9173708459598, "DestWeather": "Clear", "Dest": "Luis Munoz Marin International Airport", "FlightDelayType": "No Delay", "OriginCountry": "IT", "dayOfWeek": 0, "DistanceKilometers": 7735.755582005642, "timestamp": "2018-01-01T17:42:53", "DestLocation": {"lat": "18.43939972", "lon": "-66.00180054"}, "DestAirportID": "SJU", "Carrier": "Logstash Airways", "Cancelled": false, "FlightTimeMin": 407.1450306318759, "Origin": "Ciampino___G. B. Pastine International Airport", "OriginLocation": {"lat": "41.7994", "lon": "12.5949"}, "DestRegion": "PR-U-A", "OriginAirportID": "RM12", "OriginRegion": "IT-62", "DestCityName": "San Juan", "FlightTimeHour": 6.7857505105312645, "FlightDelayMin": 0} +{"FlightNum": "YIPS2BZ", "DestCountry": "DE", "OriginWeather": "Thunder & Lightning", "OriginCityName": "Chengdu", "AvgTicketPrice": 566.4875569256166, "DestWeather": "Sunny", "Dest": "Cologne Bonn Airport", "FlightDelayType": "No Delay", "OriginCountry": "CN", "dayOfWeek": 0, "DistanceKilometers": 7880.551894165264, "timestamp": "2018-01-01T19:55:32", "DestLocation": {"lat": "50.86589813", "lon": "7.142739773"}, "DestAirportID": "CGN", "Carrier": "Kibana Airlines", "Cancelled": true, "FlightTimeMin": 656.7126578471053, "Origin": "Chengdu Shuangliu International Airport", "OriginLocation": {"lat": "30.57850075", "lon": "103.9469986"}, "DestRegion": "DE-NW", "OriginAirportID": "CTU", "OriginRegion": "SE-BD", "DestCityName": "Cologne", "FlightTimeHour": 10.945210964118422, "FlightDelayMin": 0} +{"FlightNum": "C7IBZ42", "DestCountry": "IT", "OriginWeather": "Thunder & Lightning", "OriginCityName": "Mexico City", "AvgTicketPrice": 989.9527866266118, "DestWeather": "Damaging Wind", "Dest": "Venice Marco Polo Airport", "FlightDelayType": "No Delay", "OriginCountry": "MX", "dayOfWeek": 0, "DistanceKilometers": 10049.394341914194, "timestamp": "2018-01-01T07:49:27", "DestLocation": {"lat": "45.505299", "lon": "12.3519"}, "DestAirportID": "VE05", "Carrier": "Logstash Airways", "Cancelled": true, "FlightTimeMin": 773.0303339933996, "Origin": "Licenciado Benito Juarez International Airport", "OriginLocation": {"lat": "19.4363", "lon": "-99.072098"}, "DestRegion": "IT-34", "OriginAirportID": "AICM", "OriginRegion": "MX-DIF", "DestCityName": "Venice", "FlightTimeHour": 12.883838899889993, "FlightDelayMin": 0} +{"FlightNum": "7TTZM4I", "DestCountry": "AR", "OriginWeather": "Rain", "OriginCityName": "Cleveland", "AvgTicketPrice": 569.6132552035547, "DestWeather": "Cloudy", "Dest": "Ministro Pistarini International Airport", "FlightDelayType": "NAS Delay", "OriginCountry": "US", "dayOfWeek": 0, "DistanceKilometers": 8771.31996172178, "timestamp": "2018-01-01T01:30:47", "DestLocation": {"lat": "-34.8222", "lon": "-58.5358"}, "DestAirportID": "EZE", "Carrier": "ES-Air", "Cancelled": false, "FlightTimeMin": 704.7169201324446, "Origin": "Cleveland Hopkins International Airport", "OriginLocation": {"lat": "41.4117012", "lon": "-81.84980011"}, "DestRegion": "SE-BD", "OriginAirportID": "CLE", "OriginRegion": "US-OH", "DestCityName": "Buenos Aires", "FlightTimeHour": 11.745282002207409, "FlightDelayMin": 30} +{"FlightNum": "CSW89S3", "DestCountry": "CN", "OriginWeather": "Hail", "OriginCityName": "Olenegorsk", "AvgTicketPrice": 277.4297073844173, "DestWeather": "Clear", "Dest": "Shanghai Pudong International Airport", "FlightDelayType": "No Delay", "OriginCountry": "RU", "dayOfWeek": 0, "DistanceKilometers": 6763.2019332738655, "timestamp": "2018-01-01T07:58:17", "DestLocation": {"lat": "31.14340019", "lon": "121.8050003"}, "DestAirportID": "PVG", "Carrier": "ES-Air", "Cancelled": false, "FlightTimeMin": 355.95799648809816, "Origin": "Olenya Air Base", "OriginLocation": {"lat": "68.15180206", "lon": "33.46390152"}, "DestRegion": "SE-BD", "OriginAirportID": "XLMO", "OriginRegion": "RU-MUR", "DestCityName": "Shanghai", "FlightTimeHour": 5.932633274801636, "FlightDelayMin": 0} +{"FlightNum": "RBFKZBX", "DestCountry": "IN", "OriginWeather": "Cloudy", "OriginCityName": "Casper", "AvgTicketPrice": 772.1008456460212, "DestWeather": "Clear", "Dest": "Indira Gandhi International Airport", "FlightDelayType": "Late Aircraft Delay", "OriginCountry": "US", "dayOfWeek": 0, "DistanceKilometers": 12081.834801603853, "timestamp": "2018-01-01T00:02:06", "DestLocation": {"lat": "28.5665", "lon": "77.103104"}, "DestAirportID": "DEL", "Carrier": "JetBeats", "Cancelled": false, "FlightTimeMin": 875.1146751002408, "Origin": "Casper-Natrona County International Airport", "OriginLocation": {"lat": "42.90800095", "lon": "-106.4639969"}, "DestRegion": "SE-BD", "OriginAirportID": "CPR", "OriginRegion": "US-WY", "DestCityName": "New Delhi", "FlightTimeHour": 14.585244585004013, "FlightDelayMin": 120}