diff --git a/x-pack/plugins/search_connectors/server/plugin.ts b/x-pack/plugins/search_connectors/server/plugin.ts index 85565b0c961b7..373362ce328a5 100644 --- a/x-pack/plugins/search_connectors/server/plugin.ts +++ b/x-pack/plugins/search_connectors/server/plugin.ts @@ -63,11 +63,14 @@ export class SearchConnectorsPlugin // There seems to be no way to check for agentless here // So we register a task, but do not execute it this.log.debug('Registering agentless connectors infra sync task'); - this.agentlessConnectorDeploymentsSyncService.registerInfraSyncTask( - this.config, - plugins, - coreStartServices - ); + + coreStartServices.then(([coreStart, searchConnectorsPluginStartDependencies]) => { + this.agentlessConnectorDeploymentsSyncService.registerInfraSyncTask( + plugins, + coreStart, + searchConnectorsPluginStartDependencies + ); + }); return { getConnectorTypes: () => this.connectors, diff --git a/x-pack/plugins/search_connectors/server/services/index.test.ts b/x-pack/plugins/search_connectors/server/services/index.test.ts index 0c70113c8f850..38a9e20e8fc79 100644 --- a/x-pack/plugins/search_connectors/server/services/index.test.ts +++ b/x-pack/plugins/search_connectors/server/services/index.test.ts @@ -10,7 +10,13 @@ import { ElasticsearchClientMock, elasticsearchClientMock, } from '@kbn/core-elasticsearch-client-server-mocks'; -import { AgentlessConnectorsInfraService } from '.'; +import { + AgentlessConnectorsInfraService, + ConnectorMetadata, + PackagePolicyMetadata, + getConnectorsWithoutPolicies, + getPoliciesWithoutConnectors, +} from '.'; import { savedObjectsClientMock } from '@kbn/core/server/mocks'; import { MockedLogger, loggerMock } from '@kbn/logging-mocks'; import { @@ -18,8 +24,8 @@ import { createMockAgentPolicyService, } from '@kbn/fleet-plugin/server/mocks'; import { AgentPolicyServiceInterface, PackagePolicyClient } from '@kbn/fleet-plugin/server'; -import { PackagePolicy, PackagePolicyInput } from '@kbn/fleet-plugin/common'; -import { createPackagePolicyMock } from '@kbn/fleet-plugin/common/mocks'; +import { AgentPolicy, PackagePolicy, PackagePolicyInput } from '@kbn/fleet-plugin/common'; +import { createAgentPolicyMock, createPackagePolicyMock } from '@kbn/fleet-plugin/common/mocks'; describe('AgentlessConnectorsInfraService', () => { let soClient: SavedObjectsClientContract; @@ -118,7 +124,6 @@ describe('AgentlessConnectorsInfraService', () => { expect(nativeConnectors[1].service_type).toBe(mockResult.results[1].service_type); }); }); - describe('getConnectorPackagePolicies', () => { const getMockPolicyFetchAllItems = (pages: PackagePolicy[][]) => { return { @@ -164,17 +169,17 @@ describe('AgentlessConnectorsInfraService', () => { const policies = await service.getConnectorPackagePolicies(); expect(policies.length).toBe(1); - expect(policies[0].packagePolicyId).toBe(firstPackagePolicy.id); - expect(policies[0].connectorMetadata.id).toBe( + expect(policies[0].package_policy_id).toBe(firstPackagePolicy.id); + expect(policies[0].connector_metadata.id).toBe( firstPackagePolicy.inputs[0].compiled_input.connector_id ); - expect(policies[0].connectorMetadata.name).toBe( + expect(policies[0].connector_metadata.name).toBe( firstPackagePolicy.inputs[0].compiled_input.connector_name ); - expect(policies[0].connectorMetadata.service_type).toBe( + expect(policies[0].connector_metadata.service_type).toBe( firstPackagePolicy.inputs[0].compiled_input.service_type ); - expect(policies[0].agentPolicyIds).toBe(firstPackagePolicy.policy_ids); + expect(policies[0].agent_policy_ids).toBe(firstPackagePolicy.policy_ids); }); test('Lists policies if they are returned over multiple pages', async () => { @@ -215,29 +220,29 @@ describe('AgentlessConnectorsInfraService', () => { const policies = await service.getConnectorPackagePolicies(); expect(policies.length).toBe(2); - expect(policies[0].packagePolicyId).toBe(firstPackagePolicy.id); - expect(policies[0].connectorMetadata.id).toBe( + expect(policies[0].package_policy_id).toBe(firstPackagePolicy.id); + expect(policies[0].connector_metadata.id).toBe( firstPackagePolicy.inputs[0].compiled_input.connector_id ); - expect(policies[0].connectorMetadata.name).toBe( + expect(policies[0].connector_metadata.name).toBe( firstPackagePolicy.inputs[0].compiled_input.connector_name ); - expect(policies[0].connectorMetadata.service_type).toBe( + expect(policies[0].connector_metadata.service_type).toBe( firstPackagePolicy.inputs[0].compiled_input.service_type ); - expect(policies[0].agentPolicyIds).toBe(firstPackagePolicy.policy_ids); + expect(policies[0].agent_policy_ids).toBe(firstPackagePolicy.policy_ids); - expect(policies[1].packagePolicyId).toBe(thirdPackagePolicy.id); - expect(policies[1].connectorMetadata.id).toBe( + expect(policies[1].package_policy_id).toBe(thirdPackagePolicy.id); + expect(policies[1].connector_metadata.id).toBe( thirdPackagePolicy.inputs[0].compiled_input.connector_id ); - expect(policies[1].connectorMetadata.name).toBe( + expect(policies[1].connector_metadata.name).toBe( thirdPackagePolicy.inputs[0].compiled_input.connector_name ); - expect(policies[1].connectorMetadata.service_type).toBe( + expect(policies[1].connector_metadata.service_type).toBe( thirdPackagePolicy.inputs[0].compiled_input.service_type ); - expect(policies[1].agentPolicyIds).toBe(thirdPackagePolicy.policy_ids); + expect(policies[1].agent_policy_ids).toBe(thirdPackagePolicy.policy_ids); }); test('Skips policies that have missing fields', async () => { @@ -274,6 +279,27 @@ describe('AgentlessConnectorsInfraService', () => { }); }); describe('deployConnector', () => { + let agentPolicy: AgentPolicy; + let sharepointOnlinePackagePolicy: PackagePolicy; + + beforeAll(() => { + agentPolicy = createAgentPolicyMock(); + + sharepointOnlinePackagePolicy = createPackagePolicyMock(); + sharepointOnlinePackagePolicy.id = 'this-is-package-policy-id'; + sharepointOnlinePackagePolicy.policy_ids = ['this-is-agent-policy-id']; + sharepointOnlinePackagePolicy.inputs = [ + { + type: 'connectors-py', + compiled_input: { + connector_id: '00000001', + connector_name: 'Sharepoint Online Production Connector', + service_type: 'sharepoint_online', + }, + } as PackagePolicyInput, + ]; + }); + test('Raises an error if connector.id is missing', async () => { const connector = { id: '', @@ -348,6 +374,7 @@ describe('AgentlessConnectorsInfraService', () => { }; const errorMessage = 'Failed to create a package policy hehe'; + agentPolicyInterface.create.mockResolvedValue(agentPolicy); packagePolicyService.create.mockImplementation(() => { throw new Error(errorMessage); }); @@ -359,5 +386,199 @@ describe('AgentlessConnectorsInfraService', () => { expect(e.message).toEqual(errorMessage); } }); + + test('Returns a created package policy when all goes well', async () => { + const connector = { + id: '000000001', + name: 'something', + service_type: 'github', + }; + + agentPolicyInterface.create.mockResolvedValue(agentPolicy); + packagePolicyService.create.mockResolvedValue(sharepointOnlinePackagePolicy); + + const result = await service.deployConnector(connector); + expect(result).toBe(sharepointOnlinePackagePolicy); + }); + }); + describe('removeDeployment', () => { + const packagePolicyId = 'this-is-package-policy-id'; + const agentPolicyId = 'this-is-agent-policy-id'; + let sharepointOnlinePackagePolicy: PackagePolicy; + + beforeAll(() => { + sharepointOnlinePackagePolicy = createPackagePolicyMock(); + sharepointOnlinePackagePolicy.id = packagePolicyId; + sharepointOnlinePackagePolicy.policy_ids = [agentPolicyId]; + sharepointOnlinePackagePolicy.inputs = [ + { + type: 'connectors-py', + compiled_input: { + connector_id: '00000001', + connector_name: 'Sharepoint Online Production Connector', + service_type: 'sharepoint_online', + }, + } as PackagePolicyInput, + ]; + }); + + test('Calls for deletion of both agent policy and package policy', async () => { + packagePolicyService.get.mockResolvedValue(sharepointOnlinePackagePolicy); + + await service.removeDeployment(packagePolicyId); + + expect(agentPolicyInterface.delete).toBeCalledWith(soClient, esClient, agentPolicyId); + expect(packagePolicyService.delete).toBeCalledWith(soClient, esClient, [packagePolicyId]); + }); + + test('Raises an error if deletion of agent policy failed', async () => { + packagePolicyService.get.mockResolvedValue(sharepointOnlinePackagePolicy); + + const errorMessage = 'Failed to create a package policy hehe'; + + agentPolicyInterface.delete.mockImplementation(() => { + throw new Error(errorMessage); + }); + + try { + await service.removeDeployment(packagePolicyId); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toEqual(errorMessage); + } + }); + + test('Raises an error if deletion of package policy failed', async () => { + packagePolicyService.get.mockResolvedValue(sharepointOnlinePackagePolicy); + + const errorMessage = 'Failed to create a package policy hehe'; + + packagePolicyService.delete.mockImplementation(() => { + throw new Error(errorMessage); + }); + + try { + await service.removeDeployment(packagePolicyId); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toEqual(errorMessage); + } + }); + + test('Raises an error if a policy is not found', async () => { + packagePolicyService.get.mockResolvedValue(null); + + try { + await service.removeDeployment(packagePolicyId); + expect(true).toBe(false); + } catch (e) { + expect(e.message).toContain('Failed to delete policy'); + expect(e.message).toContain(packagePolicyId); + } + }); + }); +}); + +describe('module', () => { + const githubConnector: ConnectorMetadata = { + id: '000001', + name: 'Github Connector', + service_type: 'github', + }; + + const sharepointConnector: ConnectorMetadata = { + id: '000002', + name: 'Sharepoint Connector', + service_type: 'sharepoint_online', + }; + + const mysqlConnector: ConnectorMetadata = { + id: '000003', + name: 'MySQL Connector', + service_type: 'mysql', + }; + + const githubPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-001', + agent_policy_ids: ['agent-package-001'], + connector_metadata: githubConnector, + }; + + const sharepointPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-002', + agent_policy_ids: ['agent-package-002'], + connector_metadata: sharepointConnector, + }; + + const mysqlPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-003', + agent_policy_ids: ['agent-package-003'], + connector_metadata: mysqlConnector, + }; + + describe('getPoliciesWithoutConnectors', () => { + test('Returns a missing policy if one is missing', async () => { + const missingPolicies = getPoliciesWithoutConnectors( + [githubPackagePolicy, sharepointPackagePolicy, mysqlPackagePolicy], + [githubConnector, sharepointConnector] + ); + + expect(missingPolicies.length).toBe(1); + expect(missingPolicies).toContain(mysqlPackagePolicy); + }); + + test('Returns empty array if no policies are missing', async () => { + const missingPolicies = getPoliciesWithoutConnectors( + [githubPackagePolicy, sharepointPackagePolicy, mysqlPackagePolicy], + [githubConnector, sharepointConnector, mysqlConnector] + ); + + expect(missingPolicies.length).toBe(0); + }); + + test('Returns all policies if all are missing', async () => { + const missingPolicies = getPoliciesWithoutConnectors( + [githubPackagePolicy, sharepointPackagePolicy, mysqlPackagePolicy], + [] + ); + + expect(missingPolicies.length).toBe(3); + expect(missingPolicies).toContain(githubPackagePolicy); + expect(missingPolicies).toContain(sharepointPackagePolicy); + expect(missingPolicies).toContain(mysqlPackagePolicy); + }); + }); + + describe('getConnectorsWithoutPolicies', () => { + test('Returns a missing policy if one is missing', async () => { + const missingConnectors = getConnectorsWithoutPolicies( + [githubPackagePolicy, sharepointPackagePolicy], + [githubConnector, sharepointConnector, mysqlConnector] + ); + + expect(missingConnectors.length).toBe(1); + expect(missingConnectors).toContain(mysqlConnector); + }); + + test('Returns empty array if no policies are missing', async () => { + const missingConnectors = getConnectorsWithoutPolicies( + [githubPackagePolicy, sharepointPackagePolicy, mysqlPackagePolicy], + [githubConnector, sharepointConnector, mysqlConnector] + ); + + expect(missingConnectors.length).toBe(0); + }); + + test('Returns all policies if all are missing', async () => { + const missingConnectors = getConnectorsWithoutPolicies( + [], + [githubConnector, sharepointConnector, mysqlConnector] + ); + + expect(missingConnectors.length).toBe(3); + expect(missingConnectors).toContain(githubConnector); + expect(missingConnectors).toContain(sharepointConnector); + expect(missingConnectors).toContain(mysqlConnector); + }); }); }); diff --git a/x-pack/plugins/search_connectors/server/services/index.ts b/x-pack/plugins/search_connectors/server/services/index.ts index 6406a64495683..faebbc040b64b 100644 --- a/x-pack/plugins/search_connectors/server/services/index.ts +++ b/x-pack/plugins/search_connectors/server/services/index.ts @@ -18,13 +18,15 @@ export interface ConnectorMetadata { } export interface PackagePolicyMetadata { - packagePolicyId: string; - agentPolicyIds: string[]; - connectorMetadata: ConnectorMetadata; + package_policy_id: string; + agent_policy_ids: string[]; + connector_metadata: ConnectorMetadata; } const connectorsInputName = 'connectors-py'; const pkgName = 'elastic_connectors'; +const pkgVersion = '0.0.4'; +const pkgTitle = 'Elastic Connectors'; export class AgentlessConnectorsInfraService { private logger: Logger; @@ -98,10 +100,15 @@ export class AgentlessConnectorsInfraService { // No need to skip, that's fine } + if (policy.supports_agentless !== true) { + this.logger.debug(`Policy ${policy.id} does not support agentless, skipping`); + continue; + } + policiesMetadata.push({ - packagePolicyId: policy.id, - agentPolicyIds: policy.policy_ids, - connectorMetadata: { + package_policy_id: policy.id, + agent_policy_ids: policy.policy_ids, + connector_metadata: { id: input.compiled_input.connector_id, name: input.compiled_input.connector_name || '', service_type: input.compiled_input.service_type, @@ -137,7 +144,7 @@ export class AgentlessConnectorsInfraService { const createdPolicy = await this.agentPolicyService.create(this.soClient, this.esClient, { name: `${connector.service_type} connector: ${connector.id}`, - description: 'Automatically generated', + description: `Automatically generated on ${Date.now()}`, namespace: 'default', monitoring_enabled: ['logs', 'metrics'], inactivity_timeout: 1209600, @@ -153,9 +160,9 @@ export class AgentlessConnectorsInfraService { const packagePolicy = await this.packagePolicyService.create(this.soClient, this.esClient, { policy_ids: [createdPolicy.id], package: { - title: 'Elastic Connectors', - name: 'elastic_connectors', - version: '0.0.4', + title: pkgTitle, + name: pkgName, + version: pkgVersion, }, name: `${connector.service_type} connector ${connector.id}`, description: '', @@ -163,7 +170,7 @@ export class AgentlessConnectorsInfraService { enabled: true, inputs: [ { - type: 'connectors-py', + type: connectorsInputName, enabled: true, vars: { connector_id: { type: 'string', value: connector.id }, @@ -182,13 +189,22 @@ export class AgentlessConnectorsInfraService { return packagePolicy; }; - public removeDeployment = async (policy: PackagePolicyMetadata): Promise => { - this.logger.info(`Deleting package policy ${policy.packagePolicyId}`); - await this.packagePolicyService.delete(this.soClient, this.esClient, [policy.packagePolicyId]); + public removeDeployment = async (packagePolicyId: string): Promise => { + this.logger.info(`Deleting package policy ${packagePolicyId}`); + + const policy = await this.packagePolicyService.get(this.soClient, packagePolicyId); + + if (policy == null) { + throw new Error(`Failed to delete policy ${packagePolicyId}: not found`); + } + + await this.packagePolicyService.delete(this.soClient, this.esClient, [policy.id]); + + this.logger.debug(`Deleting package policies with ids ${policy.policy_ids}`); // TODO: can we do it in one go? // Why not use deleteFleetServerPoliciesForPolicyId? - for (const agentPolicyId of policy.agentPolicyIds) { + for (const agentPolicyId of policy.policy_ids) { this.logger.info(`Deleting agent policy ${agentPolicyId}`); await this.agentPolicyService.delete(this.soClient, this.esClient, agentPolicyId); } @@ -200,7 +216,7 @@ export const getConnectorsWithoutPolicies = ( connectors: ConnectorMetadata[] ): ConnectorMetadata[] => { return connectors.filter( - (x) => packagePolicies.filter((y) => y.connectorMetadata.id === x.id).length === 0 + (x) => packagePolicies.filter((y) => y.connector_metadata.id === x.id).length === 0 ); }; @@ -209,6 +225,6 @@ export const getPoliciesWithoutConnectors = ( connectors: ConnectorMetadata[] ): PackagePolicyMetadata[] => { return packagePolicies.filter( - (x) => connectors.filter((y) => y.id === x.connectorMetadata.id).length === 0 + (x) => connectors.filter((y) => y.id === x.connector_metadata.id).length === 0 ); }; diff --git a/x-pack/plugins/search_connectors/server/task.test.ts b/x-pack/plugins/search_connectors/server/task.test.ts new file mode 100644 index 0000000000000..c77f443f9d7ed --- /dev/null +++ b/x-pack/plugins/search_connectors/server/task.test.ts @@ -0,0 +1,241 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import { infraSyncTaskRunner } from './task'; +import { ConcreteTaskInstance, TaskStatus } from '@kbn/task-manager-plugin/server'; +import { + AgentlessConnectorsInfraService, + ConnectorMetadata, + PackagePolicyMetadata, +} from './services'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import { createPackagePolicyMock } from '@kbn/fleet-plugin/common/mocks'; + +const DATE_1970 = '1970-01-01T00:00:00.000Z'; + +describe('infraSyncTaskRunner', () => { + const githubConnector: ConnectorMetadata = { + id: '000001', + name: 'Github Connector', + service_type: 'github', + }; + + const sharepointConnector: ConnectorMetadata = { + id: '000002', + name: 'Sharepoint Connector', + service_type: 'sharepoint_online', + }; + + const mysqlConnector: ConnectorMetadata = { + id: '000003', + name: 'MySQL Connector', + service_type: 'mysql', + }; + + const githubPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-001', + agent_policy_ids: ['agent-package-001'], + connector_metadata: githubConnector, + }; + + const sharepointPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-002', + agent_policy_ids: ['agent-package-002'], + connector_metadata: sharepointConnector, + }; + + const mysqlPackagePolicy: PackagePolicyMetadata = { + package_policy_id: 'agent-003', + agent_policy_ids: ['agent-package-003'], + connector_metadata: mysqlConnector, + }; + + let logger: MockedLogger; + let serviceMock: jest.Mocked; + let licensePluginStartMock: jest.Mocked; + + const taskInstanceStub: ConcreteTaskInstance = { + id: '', + attempts: 0, + status: TaskStatus.Running, + version: '123', + runAt: new Date(), + scheduledAt: new Date(), + startedAt: new Date(DATE_1970), + retryAt: new Date(Date.now() + 5 * 60 * 1000), + state: {}, + taskType: 'backfill', + timeoutOverride: '3m', + params: { + adHocRunParamsId: 'abc', + spaceId: 'default', + }, + ownerId: null, + }; + + const invalidLicenseMock = licensingMock.createLicenseMock(); + invalidLicenseMock.check.mockReturnValue({ state: 'invalid' }); + + const validLicenseMock = licensingMock.createLicenseMock(); + validLicenseMock.check.mockReturnValue({ state: 'valid' }); + + beforeAll(async () => { + logger = loggerMock.create(); + serviceMock = { + getNativeConnectors: jest.fn(), + getConnectorPackagePolicies: jest.fn(), + deployConnector: jest.fn(), + removeDeployment: jest.fn(), + } as unknown as jest.Mocked; + + licensePluginStartMock = { + getLicense: jest.fn(), + } as unknown as jest.Mocked; + }); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('Does nothing if no connectors or policies are configured', async () => { + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.deployConnector).not.toBeCalled(); + expect(serviceMock.removeDeployment).not.toBeCalled(); + }); + + test('Does nothing if connectors or policies requires deployment but license is not supported', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([mysqlConnector, githubConnector]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([sharepointPackagePolicy]); + licensePluginStartMock.getLicense.mockResolvedValue(invalidLicenseMock); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.deployConnector).not.toBeCalled(); + expect(serviceMock.removeDeployment).not.toBeCalled(); + expect(logger.warn).toBeCalledWith(expect.stringMatching(/.*not compatible.*/)); + expect(logger.warn).toBeCalledWith(expect.stringMatching(/.*license.*/)); + }); + + test('Does nothing if all connectors and package policies are in-sync', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([ + mysqlConnector, + githubConnector, + sharepointConnector, + ]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([ + mysqlPackagePolicy, + githubPackagePolicy, + sharepointPackagePolicy, + ]); + licensePluginStartMock.getLicense.mockResolvedValue(validLicenseMock); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.deployConnector).not.toBeCalled(); + expect(serviceMock.removeDeployment).not.toBeCalled(); + expect(logger.warn).not.toBeCalled(); + }); + + test('Deploys connectors if no policies has been created for these connectors', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([mysqlConnector, githubConnector]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([sharepointPackagePolicy]); + licensePluginStartMock.getLicense.mockResolvedValue(validLicenseMock); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.deployConnector).toBeCalledWith(mysqlConnector); + expect(serviceMock.deployConnector).toBeCalledWith(githubConnector); + }); + + test('Deploys connectors even if another connectors failed to be deployed', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([ + mysqlConnector, + githubConnector, + sharepointConnector, + ]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([]); + licensePluginStartMock.getLicense.mockResolvedValue(validLicenseMock); + serviceMock.deployConnector.mockImplementation(async (connector) => { + if (connector === mysqlConnector || connector === githubConnector) { + throw new Error('Cannot deploy these connectors'); + } + + return createPackagePolicyMock(); + }); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.deployConnector).toBeCalledWith(mysqlConnector); + expect(serviceMock.deployConnector).toBeCalledWith(githubConnector); + expect(serviceMock.deployConnector).toBeCalledWith(sharepointConnector); + }); + + test('Removes a package policy if no connectors match the policy', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([mysqlConnector, githubConnector]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([sharepointPackagePolicy]); + licensePluginStartMock.getLicense.mockResolvedValue(validLicenseMock); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.removeDeployment).toBeCalledWith(sharepointPackagePolicy.package_policy_id); + }); + + test('Removes deployments even if another connectors failed to be undeployed', async () => { + serviceMock.getNativeConnectors.mockResolvedValue([]); + serviceMock.getConnectorPackagePolicies.mockResolvedValue([ + sharepointPackagePolicy, + mysqlPackagePolicy, + githubPackagePolicy, + ]); + licensePluginStartMock.getLicense.mockResolvedValue(validLicenseMock); + serviceMock.removeDeployment.mockImplementation(async (policyId) => { + if ( + policyId === sharepointPackagePolicy.package_policy_id || + policyId === mysqlPackagePolicy.package_policy_id + ) { + throw new Error('Cannot deploy these connectors'); + } + }); + + await infraSyncTaskRunner( + logger, + serviceMock, + licensePluginStartMock + )({ taskInstance: taskInstanceStub }).run(); + + expect(serviceMock.removeDeployment).toBeCalledWith(sharepointPackagePolicy.package_policy_id); + expect(serviceMock.removeDeployment).toBeCalledWith(mysqlPackagePolicy.package_policy_id); + expect(serviceMock.removeDeployment).toBeCalledWith(githubPackagePolicy.package_policy_id); + }); +}); diff --git a/x-pack/plugins/search_connectors/server/task.ts b/x-pack/plugins/search_connectors/server/task.ts index 3f1ac8451d5b8..32e3dc202b902 100644 --- a/x-pack/plugins/search_connectors/server/task.ts +++ b/x-pack/plugins/search_connectors/server/task.ts @@ -13,9 +13,9 @@ import type { TaskInstance, } from '@kbn/task-manager-plugin/server'; +import { LicensingPluginStart } from '@kbn/licensing-plugin/server'; import type { SearchConnectorsPluginStartDependencies, - SearchConnectorsPluginStart, SearchConnectorsPluginSetupDependencies, } from './types'; import { @@ -39,6 +39,95 @@ const AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_TYPE = 'search:agentless-connect const SCHEDULE = { interval: '10s' }; +export function infraSyncTaskRunner( + logger: Logger, + service: AgentlessConnectorsInfraService, + licensingPluginStart: LicensingPluginStart +) { + return ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + run: async () => { + try { + // We fetch some info even if license does not permit actual operations. + // This is done so that we could give a warning to the user only + // if they are actually using the feature. + logger.debug('Checking state of connectors and agentless policies'); + + // Fetch connectors + const nativeConnectors = await service.getNativeConnectors(); + + const policiesMetadata = await service.getConnectorPackagePolicies(); + + // Check license if any native connectors or agentless policies found + if (nativeConnectors.length > 0 || policiesMetadata.length > 0) { + const license = await licensingPluginStart.getLicense(); + + if (license.check('fleet', 'platinum').state !== 'valid') { + logger.warn( + 'Current license is not compatible with agentless connectors. Please upgrade the license to at least platinum' + ); + return; + } + } + + // Deploy Policies + const connectorsWithoutPolicies = getConnectorsWithoutPolicies( + policiesMetadata, + nativeConnectors + ); + + let agentlessConnectorsDeployed = 0; + for (const connectorMetadata of connectorsWithoutPolicies) { + // We try-catch to still be able to deploy other connectors if some fail + try { + await service.deployConnector(connectorMetadata); + + agentlessConnectorsDeployed += 1; + } catch (e) { + logger.warn( + `Error creating an agentless deployment for connector ${connectorMetadata.id}: ${e.message}` + ); + } + } + + // Delete policies + const policiesWithoutConnectors = getPoliciesWithoutConnectors( + policiesMetadata, + nativeConnectors + ); + let agentlessConnectorsRemoved = 0; + + for (const policyMetadata of policiesWithoutConnectors) { + // We try-catch to still be able to deploy other connectors if some fail + try { + await service.removeDeployment(policyMetadata.package_policy_id); + + agentlessConnectorsRemoved += 1; + } catch (e) { + logger.warn( + `Error when deleting a package policy ${policyMetadata.package_policy_id}: ${e.message}` + ); + } + } + return { + state: {}, + schedule: SCHEDULE, + }; + } catch (e) { + logger.warn(`Error executing agentless deployment sync task: ${e.message}`); + return { + state: {}, + schedule: SCHEDULE, + }; + } + }, + cancel: async () => { + logger.warn('timed out'); + }, + }; + }; +} + export class AgentlessConnectorDeploymentsSyncService { private logger: Logger; @@ -46,14 +135,28 @@ export class AgentlessConnectorDeploymentsSyncService { this.logger = logger; } public registerInfraSyncTask( - config: SearchConnectorsConfig, plugins: SearchConnectorsPluginSetupDependencies, - coreStartServicesPromise: Promise< - [CoreStart, SearchConnectorsPluginStartDependencies, SearchConnectorsPluginStart] - > + coreStart: CoreStart, + searchConnectorsPluginStartDependencies: SearchConnectorsPluginStartDependencies ) { const taskManager = plugins.taskManager; + const esClient = coreStart.elasticsearch.client.asInternalUser; + const savedObjects = coreStart.savedObjects; + + const agentPolicyService = searchConnectorsPluginStartDependencies.fleet.agentPolicyService; + const packagePolicyService = searchConnectorsPluginStartDependencies.fleet.packagePolicyService; + + const soClient = new SavedObjectsClient(savedObjects.createInternalRepository()); + + const service = new AgentlessConnectorsInfraService( + soClient, + esClient, + packagePolicyService, + agentPolicyService, + this.logger + ); + taskManager.registerTaskDefinitions({ [AGENTLESS_CONNECTOR_DEPLOYMENTS_SYNC_TASK_TYPE]: { title: 'Agentless Connector Deployment Check and Sync', @@ -61,89 +164,11 @@ export class AgentlessConnectorDeploymentsSyncService { 'This task peridocally checks native connectors, agent policies and syncs them if they are out of sync', timeout: '1m', maxAttempts: 3, - createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { - return { - run: async () => { - // TODO: not run if no license - - this.logger.debug('Checking state of connectors and agentless policies'); - const startServices = await coreStartServicesPromise; - - const esClient = startServices[0].elasticsearch.client.asInternalUser; - const savedObjects = startServices[0].savedObjects; - - const agentPolicyService = startServices[1].fleet.agentPolicyService; - const packagePolicyService = startServices[1].fleet.packagePolicyService; - - const soClient = new SavedObjectsClient(savedObjects.createInternalRepository()); - - const service = new AgentlessConnectorsInfraService( - soClient, - esClient, - packagePolicyService, - agentPolicyService, - this.logger - ); - - // Fetch connectors - const nativeConnectors = await service.getNativeConnectors(); - - const policiesMetadata = await service.getConnectorPackagePolicies(); - - const connectorsWithoutPolicies = getConnectorsWithoutPolicies( - policiesMetadata, - nativeConnectors - ); - - // Check license if any native connectors or agentless policies found - if (nativeConnectors.length > 0 || policiesMetadata.length > 0) { - const license = await startServices[1].licensing.getLicense(); - - if (license.check('fleet', 'platinum').state !== 'valid') { - this.logger.warn( - 'Current license is not compatible with agentless connectors. Please upgrade the license to at least platinum' - ); - return; - } - } - - let agentlessConnectorsDeployed = 0; - for (const connectorMetadata of connectorsWithoutPolicies) { - await service.deployConnector(connectorMetadata); - - agentlessConnectorsDeployed += 1; - } - - const policiesWithoutConnectors = getPoliciesWithoutConnectors( - policiesMetadata, - nativeConnectors - ); - let agentlessConnectorsRemoved = 0; - - for (const policyMetadata of policiesWithoutConnectors) { - await service.removeDeployment(policyMetadata); - - agentlessConnectorsRemoved += 1; - } - - try { - return { - state: {}, - schedule: SCHEDULE, - }; - } catch (e) { - this.logger.warn(`Error executing agentless deployment sync task: ${e.message}`); - return { - state: {}, - schedule: SCHEDULE, - }; - } - }, - cancel: async () => { - this.logger.warn('timed out'); - }, - }; - }, + createTaskRunner: infraSyncTaskRunner( + this.logger, + service, + searchConnectorsPluginStartDependencies.licensing + ), }, }); }