diff --git a/x-pack/plugins/task_manager/server/config.ts b/x-pack/plugins/task_manager/server/config.ts index 044c982312635..92d7429501cd9 100644 --- a/x-pack/plugins/task_manager/server/config.ts +++ b/x-pack/plugins/task_manager/server/config.ts @@ -36,7 +36,7 @@ export const CLAIM_STRATEGY_MGET = 'mget'; export const DEFAULT_DISCOVERY_INTERVAL_MS = 1000 * 10; // 10 seconds const MIN_DISCOVERY_INTERVAL_MS = 1000; // 1 second const MAX_DISCOVERY_INTERVAL_MS = 1000 * 60 * 5; // 5 minutes -export const DISCOVERY_INTERVAL_AFTER_BLOCK_EXCEPTION_MS = 6 * 1000 * 10; // 10 seconds +export const DISCOVERY_INTERVAL_AFTER_BLOCK_EXCEPTION_MS = 6 * 1000 * 10; // 60 seconds export const DEFAULT_ACTIVE_NODES_LOOK_BACK_DURATION = '30s'; const FIVE_MIN_IN_MS = 5 * 60 * 1000; diff --git a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts index ab1d1bc0498fd..d048cc4063fa2 100644 --- a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts +++ b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts @@ -13,6 +13,7 @@ import { ADJUST_THROUGHPUT_INTERVAL } from '../lib/create_managed_configuration' import { TaskManagerPlugin, TaskManagerStartContract } from '../plugin'; import { coreMock } from '@kbn/core/server/mocks'; import { TaskManagerConfig } from '../config'; +import { BulkUpdateError } from '../lib/bulk_update_error'; describe('managed configuration', () => { let taskManagerStart: TaskManagerStartContract; @@ -130,14 +131,41 @@ describe('managed configuration', () => { clock.tick(ADJUST_THROUGHPUT_INTERVAL); expect(logger.warn).toHaveBeenCalledWith( - 'Poll interval configuration is temporarily increased after Elasticsearch returned 1 "too many request" and/or "execute [inline] script" error(s).' + 'Poll interval configuration is temporarily increased after Elasticsearch returned 1 "too many request" and/or "execute [inline] script" and/or "cluster_block_exception" error(s).' ); expect(logger.debug).toHaveBeenCalledWith( - 'Poll interval configuration changing from 3000 to 3600 after seeing 1 "too many request" and/or "execute [inline] script" error(s)' + 'Poll interval configuration changing from 3000 to 3600 after seeing 1 "too many request" and/or "execute [inline] script" and/or "cluster_block_exception" error(s).' ); expect(logger.debug).toHaveBeenCalledWith('Task poller now using interval of 3600ms'); }); + test('should increase poll interval when Elasticsearch returns a cluster_block_exception error', async () => { + savedObjectsClient.create.mockRejectedValueOnce( + new BulkUpdateError({ + statusCode: 403, + message: 'index is blocked', + type: 'cluster_block_exception', + }) + ); + + await expect( + taskManagerStart.schedule({ + taskType: 'foo', + state: {}, + params: {}, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"index is blocked"`); + clock.tick(100000); + + expect(logger.warn).toHaveBeenCalledWith( + 'Poll interval configuration is temporarily increased after Elasticsearch returned 1 "too many request" and/or "execute [inline] script" and/or "cluster_block_exception" error(s).' + ); + expect(logger.debug).toHaveBeenCalledWith( + 'Poll interval configuration changing from 3000 to 61000 after seeing 1 "too many request" and/or "execute [inline] script" and/or "cluster_block_exception" error(s).' + ); + expect(logger.debug).toHaveBeenCalledWith('Task poller now using interval of 61000ms'); + }); + test('should increase poll interval when Elasticsearch returns "cannot execute [inline] scripts" error', async () => { const childEsClient = esStart.client.asInternalUser.child({}) as jest.Mocked; childEsClient.search.mockImplementationOnce(async () => { @@ -151,10 +179,10 @@ describe('managed configuration', () => { clock.tick(ADJUST_THROUGHPUT_INTERVAL); expect(logger.warn).toHaveBeenCalledWith( - 'Poll interval configuration is temporarily increased after Elasticsearch returned 1 "too many request" and/or "execute [inline] script" error(s).' + 'Poll interval configuration is temporarily increased after Elasticsearch returned 1 "too many request" and/or "execute [inline] script" and/or "cluster_block_exception" error(s).' ); expect(logger.debug).toHaveBeenCalledWith( - 'Poll interval configuration changing from 3000 to 3600 after seeing 1 "too many request" and/or "execute [inline] script" error(s)' + 'Poll interval configuration changing from 3000 to 3600 after seeing 1 "too many request" and/or "execute [inline] script" and/or "cluster_block_exception" error(s).' ); expect(logger.debug).toHaveBeenCalledWith('Task poller now using interval of 3600ms'); }); diff --git a/x-pack/plugins/task_manager/server/lib/create_managed_configuration.test.ts b/x-pack/plugins/task_manager/server/lib/create_managed_configuration.test.ts index 501d35e2c8550..cd13ac20026ed 100644 --- a/x-pack/plugins/task_manager/server/lib/create_managed_configuration.test.ts +++ b/x-pack/plugins/task_manager/server/lib/create_managed_configuration.test.ts @@ -443,7 +443,7 @@ describe('createManagedConfiguration()', () => { errors$.next(SavedObjectsErrorHelpers.createTooManyRequestsError('a', 'b')); clock.tick(ADJUST_THROUGHPUT_INTERVAL); expect(logger.warn).toHaveBeenCalledWith( - 'Poll interval configuration is temporarily increased after Elasticsearch returned 1 "too many request" and/or "execute [inline] script and/or "cluster_block_exception"" error(s).' + 'Poll interval configuration is temporarily increased after Elasticsearch returned 1 "too many request" and/or "execute [inline] script" and/or "cluster_block_exception" error(s).' ); }); diff --git a/x-pack/plugins/task_manager/server/lib/create_managed_configuration.ts b/x-pack/plugins/task_manager/server/lib/create_managed_configuration.ts index c62e18b6446bd..dfe5097567dc1 100644 --- a/x-pack/plugins/task_manager/server/lib/create_managed_configuration.ts +++ b/x-pack/plugins/task_manager/server/lib/create_managed_configuration.ts @@ -141,14 +141,16 @@ function createPollIntervalScan(logger: Logger, startingPollInterval: number) { } } else { if (previousPollInterval === INTERVAL_AFTER_BLOCK_EXCEPTION) { - previousPollInterval = startingPollInterval; + newPollInterval = startingPollInterval; + } else { + // Decrease poll interval by POLL_INTERVAL_DECREASE_PERCENTAGE and use Math.floor to + // make sure the number is different than previous while not being a decimal value. + newPollInterval = Math.max( + startingPollInterval, + Math.floor(previousPollInterval * POLL_INTERVAL_DECREASE_PERCENTAGE) + ); } - // Decrease poll interval by POLL_INTERVAL_DECREASE_PERCENTAGE and use Math.floor to - // make sure the number is different than previous while not being a decimal value. - newPollInterval = Math.max( - startingPollInterval, - Math.floor(previousPollInterval * POLL_INTERVAL_DECREASE_PERCENTAGE) - ); + if (!Number.isSafeInteger(newPollInterval) || newPollInterval < 0) { logger.error( `Poll interval configuration had an issue calculating the new poll interval: Math.max(${startingPollInterval}, Math.floor(${previousPollInterval} * ${POLL_INTERVAL_DECREASE_PERCENTAGE})) = ${newPollInterval}, will keep the poll interval unchanged (${previousPollInterval})` @@ -160,11 +162,11 @@ function createPollIntervalScan(logger: Logger, startingPollInterval: number) { if (newPollInterval !== previousPollInterval) { logger.debug( - `Poll interval configuration changing from ${previousPollInterval} to ${newPollInterval} after seeing ${errorCount} "too many request" and/or "execute [inline] script" and/or "cluster_block_exception" error(s)` + `Poll interval configuration changing from ${previousPollInterval} to ${newPollInterval} after seeing ${errorCount} "too many request" and/or "execute [inline] script" and/or "cluster_block_exception" error(s).` ); if (previousPollInterval === startingPollInterval) { logger.warn( - `Poll interval configuration is temporarily increased after Elasticsearch returned ${errorCount} "too many request" and/or "execute [inline] script and/or "cluster_block_exception"" error(s).` + `Poll interval configuration is temporarily increased after Elasticsearch returned ${errorCount} "too many request" and/or "execute [inline] script" and/or "cluster_block_exception" error(s).` ); } }