feat(table-fromFlux-parser): adds a data transformation layer to the …

…UI that uses fromFlux to parse CSVs for Tables (influxdata#85)

src/shared/components/FluxTablesTransform.tsx

@@ -3,7 +3,11 @@ import {useMemo, FunctionComponent} from 'react'
 import {flatMap} from 'lodash'
 
 // Utils
-import {parseResponse} from 'src/shared/parsing/flux/response'
+import {
+ parseResponse,
+ parseResponseWithFromFlux,
+} from 'src/shared/parsing/flux/response'
+import {isFlagEnabled} from 'src/shared/utils/featureFlag'
 
 // Types
 import {FluxTable} from 'src/types'
@@ -14,7 +18,11 @@ interface Props {
 }
 
 const FluxTablesTransform: FunctionComponent<Props> = ({files, children}) => {
- const tables = useMemo(() => flatMap(files, parseResponse), [files])
+ let parserFunction = parseResponse
+ if (isFlagEnabled('fromFluxTableParser')) {
+ parserFunction = parseResponseWithFromFlux
+ }
+ const tables = useMemo(() => flatMap(files, parserFunction), [files])
  return children(tables)
 }
 

src/shared/parsing/flux/constants.ts

@@ -67,7 +67,7 @@ export const MEASUREMENTS_RESPONSE = `#datatype,string,long,dateTime:RFC3339,dat
 
 `
 
-/* 
+/*
 From the following request:
 
  from(db: "telegraf")
@@ -119,6 +119,44 @@ export const MULTI_SCHEMA_RESPONSE = `#datatype,string,long,dateTime:RFC3339,dat
 
 
 `
+export const MULTI_YIELD_CSV = `#group,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false
+#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,double,dateTime:RFC3339
+#default,mean,,,,,,,,,,,,,,,,,,,,,,,
+,result,table,_start,_stop,_field,_measurement,com.docker.compose.config-hash,com.docker.compose.container-number,com.docker.compose.oneoff,com.docker.compose.project,com.docker.compose.project.config_files,com.docker.compose.project.working_dir,com.docker.compose.service,com.docker.compose.version,container_image,container_name,container_status,container_version,cpu,engine_host,host,server_version,_value,_time
+,,0,2020-10-06T20:55:09.9842563Z,2020-10-06T21:00:09.9842563Z,usage_percent,docker_container_cpu,f8671f6468549a5e81739babf890629f328ca986e1ed1bd5fe7f2529250e812f,1,False,influx,"compose/fig.cloud.yml,compose/fig.chronograf.oss.yml,compose/fig.chronograf.cloud.yml,compose/fig.local.yml",/Users/asalem/go/src/github.com/monitor-ci/compose,influxdb,1.25.4,quay.io/influxdb/idpe-acceptance,influx_influxdb_1,running,latest,cpu-total,docker-desktop,82c47d81f234,19.03.8,2.169788866995074,2020-10-06T20:55:17.321Z
+
+#group,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false
+#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,double,dateTime:RFC3339
+#default,foo,,,,,,,,,,,,,,,,,,,,,,,
+,result,table,_start,_stop,_field,_measurement,com.docker.compose.config-hash,com.docker.compose.container-number,com.docker.compose.oneoff,com.docker.compose.project,com.docker.compose.project.config_files,com.docker.compose.project.working_dir,com.docker.compose.service,com.docker.compose.version,container_image,container_name,container_status,container_version,cpu,engine_host,host,server_version,_value,_time
+,,0,2020-10-06T20:55:09.9842563Z,2020-10-06T21:00:09.9842563Z,usage_percent,docker_container_cpu,f8671f6468549a5e81739babf890629f328ca986e1ed1bd5fe7f2529250e812f,1,False,influx,"compose/fig.cloud.yml,compose/fig.chronograf.oss.yml,compose/fig.chronograf.cloud.yml,compose/fig.local.yml",/Users/asalem/go/src/github.com/monitor-ci/compose,influxdb,1.25.4,quay.io/influxdb/idpe-acceptance,influx_influxdb_1,running,latest,cpu-total,docker-desktop,82c47d81f234,19.03.8,2.169788866995074,2020-10-06T20:55:17.321Z
+,,0,2020-10-06T20:55:09.9842563Z,2020-10-06T21:00:09.9842563Z,usage_percent,docker_container_cpu,f8671f6468549a5e81739babf890629f328ca986e1ed1bd5fe7f2529250e812f,1,False,influx,"compose/fig.cloud.yml,compose/fig.chronograf.oss.yml,compose/fig.chronograf.cloud.yml,compose/fig.local.yml",/Users/asalem/go/src/github.com/monitor-ci/compose,influxdb,1.25.4,quay.io/influxdb/idpe-acceptance,influx_influxdb_1,running,latest,cpu-total,docker-desktop,82c47d81f234,19.03.8,2.0242590121457487,2020-10-06T20:55:27.317Z
+
+#group,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false
+#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,double,dateTime:RFC3339
+#default,baz,,,,,,,,,,,,,,,,,,,,,,,
+,result,table,_start,_stop,_field,_measurement,com.docker.compose.config-hash,com.docker.compose.container-number,com.docker.compose.oneoff,com.docker.compose.project,com.docker.compose.project.config_files,com.docker.compose.project.working_dir,com.docker.compose.service,com.docker.compose.version,container_image,container_name,container_status,container_version,cpu,engine_host,host,server_version,_value,_time
+,,0,2020-10-06T20:55:09.9842563Z,2020-10-06T21:00:09.9842563Z,usage_percent,docker_container_cpu,f8671f6468549a5e81739babf890629f328ca986e1ed1bd5fe7f2529250e812f,1,False,influx,"compose/fig.cloud.yml,compose/fig.chronograf.oss.yml,compose/fig.chronograf.cloud.yml,compose/fig.local.yml",/Users/asalem/go/src/github.com/monitor-ci/compose,influxdb,1.25.4,quay.io/influxdb/idpe-acceptance,influx_influxdb_1,running,latest,cpu-total,docker-desktop,82c47d81f234,19.03.8,2.169788866995074,2020-10-06T20:55:17.321Z`
+
+export const MULTI_YIELD_AND_TABLE_CSV = `#group,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false
+#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,double,dateTime:RFC3339
+#default,mean,,,,,,,,,,,,,,,,,,,,,,,
+,result,table,_start,_stop,_field,_measurement,com.docker.compose.config-hash,com.docker.compose.container-number,com.docker.compose.oneoff,com.docker.compose.project,com.docker.compose.project.config_files,com.docker.compose.project.working_dir,com.docker.compose.service,com.docker.compose.version,container_image,container_name,container_status,container_version,cpu,engine_host,host,server_version,_value,_time
+,,0,2020-10-06T20:55:09.9842563Z,2020-10-06T21:00:09.9842563Z,usage_percent,docker_container_cpu,f8671f6468549a5e81739babf890629f328ca986e1ed1bd5fe7f2529250e812f,1,False,influx,"compose/fig.cloud.yml,compose/fig.chronograf.oss.yml,compose/fig.chronograf.cloud.yml,compose/fig.local.yml",/Users/asalem/go/src/github.com/monitor-ci/compose,influxdb,1.25.4,quay.io/influxdb/idpe-acceptance,influx_influxdb_1,running,latest,cpu-total,docker-desktop,82c47d81f234,19.03.8,2.169788866995074,2020-10-06T20:55:17.321Z
+
+#group,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false
+#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,double,dateTime:RFC3339
+#default,foo,,,,,,,,,,,,,,,,,,,,,,,
+,result,table,_start,_stop,_field,_measurement,com.docker.compose.config-hash,com.docker.compose.container-number,com.docker.compose.oneoff,com.docker.compose.project,com.docker.compose.project.config_files,com.docker.compose.project.working_dir,com.docker.compose.service,com.docker.compose.version,container_image,container_name,container_status,container_version,cpu,engine_host,host,server_version,_value,_time
+,,0,2020-10-06T20:55:09.9842563Z,2020-10-06T21:00:09.9842563Z,usage_percent,docker_container_cpu,f8671f6468549a5e81739babf890629f328ca986e1ed1bd5fe7f2529250e812f,1,False,influx,"compose/fig.cloud.yml,compose/fig.chronograf.oss.yml,compose/fig.chronograf.cloud.yml,compose/fig.local.yml",/Users/asalem/go/src/github.com/monitor-ci/compose,influxdb,1.25.4,quay.io/influxdb/idpe-acceptance,influx_influxdb_1,running,latest,cpu-total,docker-desktop,82c47d81f234,19.03.8,2.169788866995074,2020-10-06T20:55:17.321Z
+,,1,2020-10-06T20:55:09.9842563Z,2020-10-06T21:00:09.9842563Z,usage_percent,docker_container_cpu,f8671f6468549a5e81739babf890629f328ca986e1ed1bd5fe7f2529250e812f,1,False,influx,"compose/fig.cloud.yml,compose/fig.chronograf.oss.yml,compose/fig.chronograf.cloud.yml,compose/fig.local.yml",/Users/asalem/go/src/github.com/monitor-ci/compose,influxdb,1.25.4,quay.io/influxdb/idpe-acceptance,influx_influxdb_1,running,latest,cpu-total,docker-desktop,82c47d81f234,19.03.8,2.0242590121457487,2020-10-06T20:55:27.317Z
+
+#group,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false
+#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,string,double,dateTime:RFC3339
+#default,baz,,,,,,,,,,,,,,,,,,,,,,,
+,result,table,_start,_stop,_field,_measurement,com.docker.compose.config-hash,com.docker.compose.container-number,com.docker.compose.oneoff,com.docker.compose.project,com.docker.compose.project.config_files,com.docker.compose.project.working_dir,com.docker.compose.service,com.docker.compose.version,container_image,container_name,container_status,container_version,cpu,engine_host,host,server_version,_value,_time
+,,0,2020-10-06T20:55:09.9842563Z,2020-10-06T21:00:09.9842563Z,usage_percent,docker_container_cpu,f8671f6468549a5e81739babf890629f328ca986e1ed1bd5fe7f2529250e812f,1,False,influx,"compose/fig.cloud.yml,compose/fig.chronograf.oss.yml,compose/fig.chronograf.cloud.yml,compose/fig.local.yml",/Users/asalem/go/src/github.com/monitor-ci/compose,influxdb,1.25.4,quay.io/influxdb/idpe-acceptance,influx_influxdb_1,running,latest,cpu-total,docker-desktop,82c47d81f234,19.03.8,2.169788866995074,2020-10-06T20:55:17.321Z`
+
 export const MULTI_VALUE_ROW = `
 #datatype,string,long,dateTime:RFC3339,double,double,string
 #group,false,false,false,false,false,true

src/shared/parsing/flux/response.test.ts

@@ -1,8 +1,14 @@
-import {parseResponse} from 'src/shared/parsing/flux/response'
+import {
+ parseResponse,
+ parseResponseWithFromFlux,
+} from 'src/shared/parsing/flux/response'
 import {
  RESPONSE_NO_METADATA,
  RESPONSE_METADATA,
  MULTI_SCHEMA_RESPONSE,
+ MULTI_VALUE_ROW,
+ MULTI_YIELD_CSV,
+ MULTI_YIELD_AND_TABLE_CSV,
  EXPECTED_COLUMNS,
  TRUNCATED_RESPONSE,
 } from 'src/shared/parsing/flux/constants'
@@ -91,3 +97,109 @@ describe('parseResponse', () => {
  })
  })
 })
+
+describe('parseResponseWithFromFlux', () => {
+ test('parseResponseWithFromFlux splits chunks based on tables and yields', () => {
+ const result = parseResponseWithFromFlux(MULTI_SCHEMA_RESPONSE)
+ expect(result).toHaveLength(4)
+ })
+ test('parseResponseWithFromFlux splits chunks based on tables', () => {
+ const result = parseResponseWithFromFlux(MULTI_VALUE_ROW)
+ expect(result).toHaveLength(2)
+ })
+ test('parseResponseWithFromFlux splits chunks based on yields', () => {
+ const result = parseResponseWithFromFlux(MULTI_YIELD_CSV)
+ expect(result).toHaveLength(3)
+ })
+ test('parseResponseWithFromFlux splits chunks based on yields and tables', () => {
+ const result = parseResponseWithFromFlux(MULTI_YIELD_AND_TABLE_CSV)
+ expect(result).toHaveLength(4)
+ })
+
+ describe('result name', () => {
+ test('uses the result name from the result column if present', () => {
+ const resp = `#group,false,false,false,false,false,false,true,true,true
+#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,long,string,string,string
+#default,,,,,,,,,
+,result,table,_start,_stop,_time,_value,_field,_measurement,host
+,max,0,2018-12-10T18:21:52.748859Z,2018-12-10T18:30:00Z,2018-12-10T18:29:58Z,4906213376,active,mem,oox4k.local
+,max,0,2018-12-10T18:30:00Z,2018-12-10T19:00:00Z,2018-12-10T18:54:08Z,5860683776,active,mem,oox4k.local
+,max,0,2018-12-10T19:00:00Z,2018-12-10T19:21:52.748859Z,2018-12-10T19:11:58Z,5115428864,active,mem,oox4k.local
+
+`
+
+ const actual = parseResponseWithFromFlux(resp)
+
+ expect(actual[0].result).toBe('max')
+ })
+
+ test('uses the result name from the default annotation if result columns are empty', () => {
+ const resp = `#group,false,false,false,false,false,false,true,true,true
+#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,long,string,string,string
+#default,max,,,,,,,,
+,result,table,_start,_stop,_time,_value,_field,_measurement,host
+,,0,2018-12-10T18:21:52.748859Z,2018-12-10T18:30:00Z,2018-12-10T18:29:58Z,4906213376,active,mem,oox4k.local
+,,0,2018-12-10T18:30:00Z,2018-12-10T19:00:00Z,2018-12-10T18:54:08Z,5860683776,active,mem,oox4k.local
+,,0,2018-12-10T19:00:00Z,2018-12-10T19:21:52.748859Z,2018-12-10T19:11:58Z,5115428864,active,mem,oox4k.local
+
+#group,false,false,false,false,false,false,true,true,true
+#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,long,string,string,string
+#default,min,,,,,,,,
+,result,table,_start,_stop,_time,_value,_field,_measurement,host
+,,0,2018-12-10T18:21:52.748859Z,2018-12-10T18:30:00Z,2018-12-10T18:29:48Z,4589981696,active,mem,oox4k.local
+,,0,2018-12-10T18:30:00Z,2018-12-10T19:00:00Z,2018-12-10T18:40:18Z,4318040064,active,mem,oox4k.local
+,,0,2018-12-10T19:00:00Z,2018-12-10T19:21:52.748859Z,2018-12-10T19:13:58Z,4131692544,active,mem,oox4k.local
+
+
+`
+ const actual = parseResponseWithFromFlux(resp)
+
+ expect(actual).toHaveLength(2)
+ expect(actual[0].result).toBe('max')
+ expect(actual[1].result).toBe('min')
+ })
+ })
+
+ describe('headers', () => {
+ test('throws when no metadata is present', () => {
+ expect(() => {
+ parseResponseWithFromFlux(RESPONSE_NO_METADATA)
+ }).toThrow()
+ })
+
+ test('can parse headers when metadata is present', () => {
+ const actual = parseResponseWithFromFlux(RESPONSE_METADATA)[0].data[0]
+ expect(actual).toEqual(EXPECTED_COLUMNS)
+ })
+ })
+
+ describe('group key', () => {
+ test('parses the group key properly', () => {
+ const actual = parseResponseWithFromFlux(MULTI_SCHEMA_RESPONSE)[0]
+ .groupKey
+ const expected = {
+ _field: 'usage_guest',
+ _measurement: 'cpu',
+ cpu: 'cpu-total',
+ host: 'WattsInfluxDB',
+ result: '_result',
+ }
+ expect(actual).toEqual(expected)
+ })
+ test('should output the groupKey with the yield / result', () => {
+ const actual = parseResponseWithFromFlux(MULTI_YIELD_CSV)[0].groupKey
+ const expected = {
+ result: 'mean',
+ }
+ expect(actual).toEqual(expected)
+ })
+ })
+
+ describe('partial responses', () => {
+ test('should discard tables without any non-annotation rows', () => {
+ const actual = parseResponseWithFromFlux(TRUNCATED_RESPONSE)
+
+ expect(actual).toHaveLength(2)
+ })
+ })
+})