Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Reporting] Use non-deprecated csv_searchsource API in integration tests #117456

Merged
merged 6 commits into from
Nov 9, 2021
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
*/

import expect from '@kbn/expect';
import { JOB_PARAMS_RISON_CSV_DEPRECATED } from '../services/fixtures';
import { FtrProviderContext } from '../ftr_provider_context';

import { ILM_POLICY_NAME } from '../../../plugins/reporting/common/constants';
Expand All @@ -19,6 +18,16 @@ export default function ({ getService }: FtrProviderContext) {
const reportingAPI = getService('reportingAPI');
const security = getService('security');

const JOB_PARAMS_RISON_CSV =
`(columns:!(order_date,category,customer_full_name,taxful_total_price,currency)` +
`,objectType:search,searchSource:(fields:!((field:'*',include_unmapped:true))` +
`,filter:!((meta:(field:order_date,index:aac3e500-f2c7-11ea-8250-fb138aa491e7` +
`,params:()),query:(range:(order_date:(format:strict_date_optional_time` +
`,gte:'2019-06-02T12:28:40.866Z',lte:'2019-07-18T20:59:57.136Z')))))` +
`,index:aac3e500-f2c7-11ea-8250-fb138aa491e7,parent:(filter:!(),highlightAll:!t` +
`,index:aac3e500-f2c7-11ea-8250-fb138aa491e7,query:(language:kuery,query:'')` +
`,version:!t),sort:!((order_date:desc)),trackTotalHits:!t),title:'EC SEARCH from DEFAULT')`;

describe('ILM policy migration APIs', () => {
before(async () => {
await reportingAPI.initLogs();
Expand All @@ -39,9 +48,9 @@ export default function ({ getService }: FtrProviderContext) {

// try creating a report
await supertest
.post(`/api/reporting/generate/csv`)
.post(`/api/reporting/generate/csv_searchsource`)
.set('kbn-xsrf', 'xxx')
.send({ jobParams: JOB_PARAMS_RISON_CSV_DEPRECATED });
.send({ jobParams: JOB_PARAMS_RISON_CSV });

expect(await reportingAPI.checkIlmMigrationStatus()).to.eql('ok');
});
Expand All @@ -51,9 +60,9 @@ export default function ({ getService }: FtrProviderContext) {
await es.ilm.deleteLifecycle({ name: ILM_POLICY_NAME });

await supertest
.post(`/api/reporting/generate/csv`)
.post(`/api/reporting/generate/csv_searchsource`)
.set('kbn-xsrf', 'xxx')
.send({ jobParams: JOB_PARAMS_RISON_CSV_DEPRECATED });
.send({ jobParams: JOB_PARAMS_RISON_CSV });

expect(await reportingAPI.checkIlmMigrationStatus()).to.eql('policy-not-found');
// assert that migration fixes this
Expand All @@ -64,9 +73,9 @@ export default function ({ getService }: FtrProviderContext) {
it('detects when reporting indices should be migrated due to unmanaged indices', async () => {
await reportingAPI.makeAllReportingIndicesUnmanaged();
await supertest
.post(`/api/reporting/generate/csv`)
.post(`/api/reporting/generate/csv_searchsource`)
.set('kbn-xsrf', 'xxx')
.send({ jobParams: JOB_PARAMS_RISON_CSV_DEPRECATED });
.send({ jobParams: JOB_PARAMS_RISON_CSV });

expect(await reportingAPI.checkIlmMigrationStatus()).to.eql('indices-not-managed-by-policy');
// assert that migration fixes this
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,38 @@ export default function ({ getService }: FtrProviderContext) {
await esArchiver.unload(spacesSharedObjectsArchive);
});

/*
* NOTE: All timestamps in the documents are midnight UTC.
* "00:00:00.000" means the time is formatted in UTC timezone
*/
describe('CSV saved search export', () => {
const JOB_PARAMS_CSV_DEFAULT_SPACE =
`columns:!(order_date,category,customer_full_name,taxful_total_price,currency),objectType:search,searchSource:(fields:!((field:'*',include_unmapped:true))` +
`,filter:!((meta:(field:order_date,index:aac3e500-f2c7-11ea-8250-fb138aa491e7,params:()),query:(range:(order_date:(format:strict_date_optional_time,gte:'2019-06-02T12:28:40.866Z'` +
`,lte:'2019-07-18T20:59:57.136Z'))))),index:aac3e500-f2c7-11ea-8250-fb138aa491e7,parent:(filter:!(),highlightAll:!t,index:aac3e500-f2c7-11ea-8250-fb138aa491e7` +
`,query:(language:kuery,query:''),version:!t),sort:!((order_date:desc)),trackTotalHits:!t)`;

const JOB_PARAMS_CSV_NONDEFAULT_SPACE =
`columns:!(order_date,category,customer_full_name,taxful_total_price,currency),objectType:search,searchSource:(fields:!((field:'*',include_unmapped:true))` +
`,filter:!((meta:(field:order_date,index:afac7364-c755-5f5c-acd5-8ed6605c5c77,params:()),query:(range:(order_date:(format:strict_date_optional_time` +
`,gte:'2006-11-04T19:58:58.244Z',lte:'2021-11-04T18:58:58.244Z'))))),index:afac7364-c755-5f5c-acd5-8ed6605c5c77,parent:(filter:!(),highlightAll:!t` +
`,index:afac7364-c755-5f5c-acd5-8ed6605c5c77,query:(language:kuery,query:''),version:!t),sort:!((order_date:desc)),trackTotalHits:!t)`;

it('should use formats from the default space', async () => {
kibanaServer.uiSettings.update({ 'csv:separator': ',', 'dateFormat:tz': 'UTC' });
const path = await reportingAPI.postJobJSON(`/api/reporting/generate/csv`, {
jobParams: `(conflictedTypesFields:!(),fields:!(order_date,order_date,customer_full_name,taxful_total_price),indexPatternId:aac3e500-f2c7-11ea-8250-fb138aa491e7,metaFields:!(_source,_id,_type,_index,_score),objectType:search,searchRequest:(body:(_source:(includes:!(order_date,customer_full_name,taxful_total_price)),docvalue_fields:!((field:order_date,format:date_time)),query:(bool:(filter:!((match_all:()),(range:(order_date:(format:strict_date_optional_time,gte:'2019-06-11T04:49:43.495Z',lte:'2019-07-14T10:25:34.149Z')))),must:!(),must_not:!(),should:!())),script_fields:(),sort:!((order_date:(order:desc,unmapped_type:boolean))),stored_fields:!(order_date,customer_full_name,taxful_total_price),version:!t),index:'ec*'),title:'EC SEARCH')`,
const path = await reportingAPI.postJobJSON(`/api/reporting/generate/csv_searchsource`, {
jobParams: `(${JOB_PARAMS_CSV_DEFAULT_SPACE},title:'EC SEARCH')`,
});
const csv = await getCompleted$(path).toPromise();
expect(csv).to.match(
/^"order_date","order_date","customer_full_name","taxful_total_price"\n"Jul 12, 2019 @ 00:00:00.000","Jul 12, 2019 @ 00:00:00.000","Sultan Al Boone","173.96"/
);

expectSnapshot(csv.slice(0, 500)).toMatchInline(`
"\\"order_date\\",category,\\"customer_full_name\\",\\"taxful_total_price\\",currency
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Sultan Al Boone\\",174,EUR
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",\\"Pia Richards\\",\\"41.969\\",EUR
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",\\"Brigitte Meyer\\",\\"40.969\\",EUR
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",\\"Abd Mccarthy\\",\\"41.969\\",EUR
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",\\"Robert "
`);
});

it('should use formats from non-default spaces', async () => {
Expand All @@ -72,15 +94,21 @@ export default function ({ getService }: FtrProviderContext) {
'dateFormat:tz': 'US/Alaska',
});
const path = await reportingAPI.postJobJSON(
`/s/non_default_space/api/reporting/generate/csv`,
`/s/non_default_space/api/reporting/generate/csv_searchsource`,
{
jobParams: `(conflictedTypesFields:!(),fields:!(order_date,category,customer_first_name,customer_full_name,total_quantity,total_unique_products,taxless_total_price,taxful_total_price,currency),indexPatternId:'067dec90-e7ee-11ea-a730-d58e9ea7581b',metaFields:!(_source,_id,_type,_index,_score),objectType:search,searchRequest:(body:(_source:(includes:!(order_date,category,customer_first_name,customer_full_name,total_quantity,total_unique_products,taxless_total_price,taxful_total_price,currency)),docvalue_fields:!((field:order_date,format:date_time)),query:(bool:(filter:!((match_all:()),(range:(order_date:(format:strict_date_optional_time,gte:'2019-06-11T08:24:16.425Z',lte:'2019-07-13T09:31:07.520Z')))),must:!(),must_not:!(),should:!())),script_fields:(),sort:!((order_date:(order:desc,unmapped_type:boolean))),stored_fields:!(order_date,category,customer_first_name,customer_full_name,total_quantity,total_unique_products,taxless_total_price,taxful_total_price,currency),version:!t),index:'ecommerce*'),title:'Ecom Search')`,
jobParams: `(${JOB_PARAMS_CSV_NONDEFAULT_SPACE},title:'Ecom Search from Non-Default')`,
}
);
const csv = await getCompleted$(path).toPromise();
expect(csv).to.match(
/^order_date;category;customer_first_name;customer_full_name;total_quantity;total_unique_products;taxless_total_price;taxful_total_price;currency\nJul 11, 2019 @ 16:00:00.000;/
);
expectSnapshot(csv.slice(0, 500)).toMatchInline(`
"order_date;category;customer_full_name;taxful_total_price;currency
Jul 11, 2019 @ 16:00:00.000;Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories;Sultan Al Boone;174;EUR
Jul 11, 2019 @ 16:00:00.000;Women's Shoes, Women's Clothing;Pia Richards;41.969;EUR
Jul 11, 2019 @ 16:00:00.000;Women's Clothing;Brigitte Meyer;40.969;EUR
Jul 11, 2019 @ 16:00:00.000;Men's Clothing;Abd Mccarthy;41.969;EUR
Jul 11, 2019 @ 16:00:00.000;Men's Clothing;Robert Banks;36.969;EUR
Jul 11, 2019 @ 16:00:00."
`);
});

it(`should use browserTimezone in jobParams for date formatting`, async () => {
Expand All @@ -90,25 +118,35 @@ export default function ({ getService }: FtrProviderContext) {
'csv:separator': ';',
'dateFormat:tz': tzSettings,
});
const path = await reportingAPI.postJobJSON(`/api/reporting/generate/csv`, {
jobParams: `(browserTimezone:${tzParam},conflictedTypesFields:!(),fields:!(order_date,category,customer_full_name,taxful_total_price,currency),indexPatternId:aac3e500-f2c7-11ea-8250-fb138aa491e7,metaFields:!(_source,_id,_type,_index,_score),objectType:search,searchRequest:(body:(_source:(includes:!(order_date,category,customer_full_name,taxful_total_price,currency)),docvalue_fields:!((field:order_date,format:date_time)),query:(bool:(filter:!((match_all:()),(range:(order_date:(format:strict_date_optional_time,gte:'2019-05-30T05:09:59.743Z',lte:'2019-07-26T08:47:09.682Z')))),must:!(),must_not:!(),should:!())),script_fields:(),sort:!((order_date:(order:desc,unmapped_type:boolean))),stored_fields:!(order_date,category,customer_full_name,taxful_total_price,currency),version:!t),index:'ec*'),title:'EC SEARCH from DEFAULT')`,
const path = await reportingAPI.postJobJSON(`/api/reporting/generate/csv_searchsource`, {
jobParams: `(browserTimezone:${tzParam},${JOB_PARAMS_CSV_DEFAULT_SPACE},title:'EC SEARCH')`,
});

const csv = await getCompleted$(path).toPromise();
expect(csv).to.match(
/^"order_date",category,"customer_full_name","taxful_total_price",currency\n"Jul 11, 2019 @ 17:00:00.000"/
);
expectSnapshot(csv.slice(0, 500)).toMatchInline(`
"\\"order_date\\",category,\\"customer_full_name\\",\\"taxful_total_price\\",currency
\\"Jul 11, 2019 @ 17:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Sultan Al Boone\\",174,EUR
\\"Jul 11, 2019 @ 17:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",\\"Pia Richards\\",\\"41.969\\",EUR
\\"Jul 11, 2019 @ 17:00:00.000\\",\\"Women's Clothing\\",\\"Brigitte Meyer\\",\\"40.969\\",EUR
\\"Jul 11, 2019 @ 17:00:00.000\\",\\"Men's Clothing\\",\\"Abd Mccarthy\\",\\"41.969\\",EUR
\\"Jul 11, 2019 @ 17:00:00.000\\",\\"Men's Clothing\\",\\"Robert "
`);
});

it(`should default to UTC for date formatting when timezone is not known`, async () => {
kibanaServer.uiSettings.update({ 'csv:separator': ',', 'dateFormat:tz': 'Browser' });
const path = await reportingAPI.postJobJSON(`/api/reporting/generate/csv`, {
jobParams: `(conflictedTypesFields:!(),fields:!(order_date,order_date,customer_full_name,taxful_total_price),indexPatternId:aac3e500-f2c7-11ea-8250-fb138aa491e7,metaFields:!(_source,_id,_type,_index,_score),objectType:search,searchRequest:(body:(_source:(includes:!(order_date,customer_full_name,taxful_total_price)),docvalue_fields:!((field:order_date,format:date_time)),query:(bool:(filter:!((match_all:()),(range:(order_date:(format:strict_date_optional_time,gte:'2019-06-11T04:49:43.495Z',lte:'2019-07-14T10:25:34.149Z')))),must:!(),must_not:!(),should:!())),script_fields:(),sort:!((order_date:(order:desc,unmapped_type:boolean))),stored_fields:!(order_date,customer_full_name,taxful_total_price),version:!t),index:'ec*'),title:'EC SEARCH')`,
const path = await reportingAPI.postJobJSON(`/api/reporting/generate/csv_searchsource`, {
jobParams: `(${JOB_PARAMS_CSV_DEFAULT_SPACE},title:'EC SEARCH')`,
});
const csv = await getCompleted$(path).toPromise();
expect(csv).to.match(
/^"order_date","order_date","customer_full_name","taxful_total_price"\n"Jul 12, 2019 @ 00:00:00.000","Jul 12, 2019 @ 00:00:00.000","Sultan Al Boone","173.96"/
);
expectSnapshot(csv.slice(0, 500)).toMatchInline(`
"\\"order_date\\",category,\\"customer_full_name\\",\\"taxful_total_price\\",currency
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Shoes, Men's Clothing, Women's Accessories, Men's Accessories\\",\\"Sultan Al Boone\\",174,EUR
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Shoes, Women's Clothing\\",\\"Pia Richards\\",\\"41.969\\",EUR
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Women's Clothing\\",\\"Brigitte Meyer\\",\\"40.969\\",EUR
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",\\"Abd Mccarthy\\",\\"41.969\\",EUR
\\"Jul 12, 2019 @ 00:00:00.000\\",\\"Men's Clothing\\",\\"Robert "
`);
});
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ import { FtrProviderContext } from '../ftr_provider_context';
import * as GenerationUrls from '../services/generation_urls';
import { ReportingUsageStats } from '../services/usage';

const JOB_PARAMS_CSV_DEFAULT_SPACE =
`columns:!(order_date,category,customer_full_name,taxful_total_price,currency),objectType:search,searchSource:(fields:!((field:'*',include_unmapped:true))` +
`,filter:!((meta:(field:order_date,index:aac3e500-f2c7-11ea-8250-fb138aa491e7,params:()),query:(range:(order_date:(format:strict_date_optional_time,gte:'2019-06-02T12:28:40.866Z'` +
`,lte:'2019-07-18T20:59:57.136Z'))))),index:aac3e500-f2c7-11ea-8250-fb138aa491e7,parent:(filter:!(),highlightAll:!t,index:aac3e500-f2c7-11ea-8250-fb138aa491e7` +
`,query:(language:kuery,query:''),version:!t),sort:!((order_date:desc)),trackTotalHits:!t)`;
const OSS_KIBANA_ARCHIVE_PATH = 'test/functional/fixtures/es_archiver/dashboard/current/kibana';
const OSS_DATA_ARCHIVE_PATH = 'test/functional/fixtures/es_archiver/dashboard/current/data';

Expand All @@ -20,30 +25,14 @@ interface UsageStats {
// eslint-disable-next-line import/no-default-export
export default function ({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const kibanaServer = getService('kibanaServer');
const reportingAPI = getService('reportingAPI');
const retry = getService('retry');
const usageAPI = getService('usageAPI');

describe('Usage', () => {
before(async () => {
await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH);
await esArchiver.load(OSS_DATA_ARCHIVE_PATH);

await kibanaServer.uiSettings.update({
defaultIndex: '0bf35f60-3dc9-11e8-8660-4d65aa086b3c',
});
await reportingAPI.deleteAllReports();
});

after(async () => {
await esArchiver.unload(OSS_KIBANA_ARCHIVE_PATH);
await esArchiver.unload(OSS_DATA_ARCHIVE_PATH);
});

afterEach(async () => {
await reportingAPI.deleteAllReports();
});
const deleteAllReports = () => reportingAPI.deleteAllReports();
beforeEach(deleteAllReports);
after(deleteAllReports);

describe('initial state', () => {
let usage: UsageStats;
Expand All @@ -69,8 +58,8 @@ export default function ({ getService }: FtrProviderContext) {
reportingAPI.expectAllTimePdfLayoutStats(usage, 'preserve_layout', 0);
reportingAPI.expectAllTimePdfLayoutStats(usage, 'print', 0);
reportingAPI.expectRecentPdfLayoutStats(usage, 'print', 0);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'csv', 0);
reportingAPI.expectAllTimeJobTypeTotalStats(usage, 'csv', 0);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'csv_searchsource', 0);
reportingAPI.expectAllTimeJobTypeTotalStats(usage, 'csv_searchsource', 0);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'printable_pdf', 0);
reportingAPI.expectAllTimeJobTypeTotalStats(usage, 'printable_pdf', 0);
});
Expand Down Expand Up @@ -123,10 +112,24 @@ export default function ({ getService }: FtrProviderContext) {
});

describe('from new jobs posted', () => {
it('should handle csv', async () => {
before(async () => {
await esArchiver.load(OSS_KIBANA_ARCHIVE_PATH);
await esArchiver.load(OSS_DATA_ARCHIVE_PATH);
await reportingAPI.initEcommerce();
});

after(async () => {
await esArchiver.unload(OSS_KIBANA_ARCHIVE_PATH);
await esArchiver.unload(OSS_DATA_ARCHIVE_PATH);
await reportingAPI.teardownEcommerce();
});

it('should handle csv_searchsource', async () => {
await reportingAPI.expectAllJobsToFinishSuccessfully(
await Promise.all([
reportingAPI.postJob(GenerationUrls.CSV_DISCOVER_KUERY_AND_FILTER_6_3),
reportingAPI.postJob(
`/api/reporting/generate/csv_searchsource?jobParams=(${JOB_PARAMS_CSV_DEFAULT_SPACE})`
),
])
);

Expand All @@ -135,7 +138,7 @@ export default function ({ getService }: FtrProviderContext) {
reportingAPI.expectRecentPdfAppStats(usage, 'dashboard', 0);
reportingAPI.expectRecentPdfLayoutStats(usage, 'preserve_layout', 0);
reportingAPI.expectRecentPdfLayoutStats(usage, 'print', 0);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'csv', 1);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'csv_searchsource', 1);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'printable_pdf', 0);
});

Expand All @@ -152,7 +155,7 @@ export default function ({ getService }: FtrProviderContext) {
reportingAPI.expectRecentPdfAppStats(usage, 'dashboard', 1);
reportingAPI.expectRecentPdfLayoutStats(usage, 'preserve_layout', 2);
reportingAPI.expectRecentPdfLayoutStats(usage, 'print', 0);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'csv', 0);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'csv_searchsource', 0);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'printable_pdf', 2);
});

Expand All @@ -171,14 +174,14 @@ export default function ({ getService }: FtrProviderContext) {
reportingAPI.expectRecentPdfAppStats(usage, 'dashboard', 1);
reportingAPI.expectRecentPdfLayoutStats(usage, 'preserve_layout', 0);
reportingAPI.expectRecentPdfLayoutStats(usage, 'print', 2);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'csv', 0);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'csv_searchsource', 0);
reportingAPI.expectRecentJobTypeTotalStats(usage, 'printable_pdf', 2);

reportingAPI.expectAllTimePdfAppStats(usage, 'visualization', 1);
reportingAPI.expectAllTimePdfAppStats(usage, 'dashboard', 1);
reportingAPI.expectAllTimePdfLayoutStats(usage, 'preserve_layout', 0);
reportingAPI.expectAllTimePdfLayoutStats(usage, 'print', 2);
reportingAPI.expectAllTimeJobTypeTotalStats(usage, 'csv', 0);
reportingAPI.expectAllTimeJobTypeTotalStats(usage, 'csv_searchsource', 0);
reportingAPI.expectAllTimeJobTypeTotalStats(usage, 'printable_pdf', 2);
});
});
Expand Down
Loading