[Observability Onboarding] Fix kustomize command (#199758)

## Summary Closes #199754 by introducing more fine-grained version getters to get the base version (to resolve to the git tag in the repo), the actual version which might include the IAR suffix and the docker version which replaces + with . For testing, EA host and kubernetes as well as OTel host should be tested. See here: #199758 (comment) --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Nov 14, 2024 · 7ef9773 · 7ef9773
1 parent ac2e29b
commit 7ef9773
Show file tree

Hide file tree

Showing 14 changed files with 118 additions and 41 deletions.
diff --git a/x-pack/plugins/fleet/server/services/agents/agent_service.mock.ts b/x-pack/plugins/fleet/server/services/agents/agent_service.mock.ts
@@ -15,6 +15,8 @@ const createClientMock = (): jest.Mocked<AgentClient> => ({
   getAgentStatusForAgentPolicy: jest.fn(),
   listAgents: jest.fn(),
   getLatestAgentAvailableVersion: jest.fn(),
+  getLatestAgentAvailableBaseVersion: jest.fn(),
+  getLatestAgentAvailableDockerImageVersion: jest.fn(),
   getByIds: jest.fn(async (..._) => []),
 });
 

diff --git a/x-pack/plugins/fleet/server/services/agents/agent_service.test.ts b/x-pack/plugins/fleet/server/services/agents/agent_service.test.ts
@@ -267,4 +267,28 @@ function expectApisToCallServicesSuccessfully(
     );
     expect(mockgetLatestAvailableAgentVersion).toHaveBeenCalledTimes(1);
   });
+
+  test('client.getLatestAgentAvailableBaseVersion strips away IAR suffix', async () => {
+    mockgetLatestAvailableAgentVersion.mockResolvedValue('1.2.3+build12345678987654321');
+    await expect(agentClient.getLatestAgentAvailableBaseVersion()).resolves.toEqual('1.2.3');
+  });
+
+  test('client.getLatestAgentAvailableBaseVersion does not break on usual version numbers', async () => {
+    mockgetLatestAvailableAgentVersion.mockResolvedValue('8.17.0');
+    await expect(agentClient.getLatestAgentAvailableBaseVersion()).resolves.toEqual('8.17.0');
+  });
+
+  test('client.getLatestAgentAvailableDockerImageVersion transforms IAR suffix', async () => {
+    mockgetLatestAvailableAgentVersion.mockResolvedValue('1.2.3+build12345678987654321');
+    await expect(agentClient.getLatestAgentAvailableDockerImageVersion()).resolves.toEqual(
+      '1.2.3.build12345678987654321'
+    );
+  });
+
+  test('client.getLatestAgentAvailableDockerImageVersion does not break on usual version numbers', async () => {
+    mockgetLatestAvailableAgentVersion.mockResolvedValue('8.17.0');
+    await expect(agentClient.getLatestAgentAvailableDockerImageVersion()).resolves.toEqual(
+      '8.17.0'
+    );
+  });
 }
diff --git a/x-pack/plugins/fleet/server/services/agents/agent_service.ts b/x-pack/plugins/fleet/server/services/agents/agent_service.ts
@@ -108,6 +108,14 @@ export interface AgentClient {
    * Return the latest agent available version
    */
   getLatestAgentAvailableVersion(includeCurrentVersion?: boolean): Promise<string>;
+  /**
+   * Return the latest agent available version, not taking into account IAR versions
+   */
+  getLatestAgentAvailableBaseVersion(includeCurrentVersion?: boolean): Promise<string>;
+  /**
+   * Return the latest agent available version formatted for the docker image
+   */
+  getLatestAgentAvailableDockerImageVersion(includeCurrentVersion?: boolean): Promise<string>;
 }
 
 /**
@@ -163,6 +171,16 @@ class AgentClientImpl implements AgentClient {
     );
   }
 
+  public async getLatestAgentAvailableBaseVersion(includeCurrentVersion?: boolean) {
+    const fullVersion = await this.getLatestAgentAvailableVersion(includeCurrentVersion);
+    return fullVersion.split('+')[0];
+  }
+
+  public async getLatestAgentAvailableDockerImageVersion(includeCurrentVersion?: boolean) {
+    const fullVersion = await this.getLatestAgentAvailableVersion(includeCurrentVersion);
+    return fullVersion.replace('+', '.');
+  }
+
   public async getLatestAgentAvailableVersion(includeCurrentVersion?: boolean) {
     await this.#runPreflight();
     return getLatestAvailableAgentVersion({ includeCurrentVersion });

diff --git a/x-pack/plugins/observability_solution/observability_onboarding/common/types.ts b/x-pack/plugins/observability_solution/observability_onboarding/common/types.ts
@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface ElasticAgentVersionInfo {
+  agentVersion: string;
+  agentBaseVersion: string;
+  agentDockerImageVersion: string;
+}
diff --git a/...ty_onboarding/public/application/quickstart_flows/auto_detect/get_auto_detect_command.tsx b/...ty_onboarding/public/application/quickstart_flows/auto_detect/get_auto_detect_command.tsx
@@ -19,7 +19,7 @@ export function getAutoDetectCommand(
       --kibana-url=${options.kibanaUrl}
       --install-key=${options.installApiKey}
       --ingest-key=${options.ingestApiKey}
-      --ea-version=${options.elasticAgentVersion}
+      --ea-version=${options.elasticAgentVersionInfo.agentVersion}
   `;
 }
 function oneLine(parts: TemplateStringsArray, ...args: string[]) {

diff --git a/...lity_onboarding/public/application/quickstart_flows/custom_logs/install_elastic_agent.tsx b/...lity_onboarding/public/application/quickstart_flows/custom_logs/install_elastic_agent.tsx
@@ -312,7 +312,7 @@ export function InstallElasticAgent() {
           apiKeyEncoded,
           apiEndpoint: setup?.apiEndpoint,
           scriptDownloadUrl: setup?.scriptDownloadUrl,
-          elasticAgentVersion: setup?.elasticAgentVersion,
+          elasticAgentVersion: setup?.elasticAgentVersionInfo.agentVersion,
           autoDownloadConfig,
           onboardingId,
         })}

diff --git a/...bility_onboarding/public/application/quickstart_flows/kubernetes/build_kubectl_command.ts b/...bility_onboarding/public/application/quickstart_flows/kubernetes/build_kubectl_command.ts
@@ -5,11 +5,13 @@
  * 2.0.
  */
 
+import { ElasticAgentVersionInfo } from '../../../../common/types';
+
 interface Params {
   encodedApiKey: string;
   onboardingId: string;
   elasticsearchUrl: string;
-  elasticAgentVersion: string;
+  elasticAgentVersionInfo: ElasticAgentVersionInfo;
 }
 
 const KUSTOMIZE_TEMPLATE_URL =
@@ -19,16 +21,16 @@ export function buildKubectlCommand({
   encodedApiKey,
   onboardingId,
   elasticsearchUrl,
-  elasticAgentVersion,
+  elasticAgentVersionInfo,
 }: Params) {
   const escapedElasticsearchUrl = elasticsearchUrl.replace(/\//g, '\\/');
 
   return `
-    kubectl kustomize ${KUSTOMIZE_TEMPLATE_URL}\\?ref\\=v${elasticAgentVersion}
+    kubectl kustomize ${KUSTOMIZE_TEMPLATE_URL}\\?ref\\=v${elasticAgentVersionInfo.agentBaseVersion}
       | sed -e 's/JUFQSV9LRVkl/${encodedApiKey}/g'
             -e "s/%ES_HOST%/${escapedElasticsearchUrl}/g"
             -e "s/%ONBOARDING_ID%/${onboardingId}/g"
-            -e "s/\\(docker.elastic.co\\/beats\\/elastic-agent\:\\).*$/\\1${elasticAgentVersion}/g"
+            -e "s/\\(docker.elastic.co\\/beats\\/elastic-agent\:\\).*$/\\1${elasticAgentVersionInfo.agentDockerImageVersion}/g"
             -e  "/{CA_TRUSTED}/c\\ "
       |  kubectl apply -f-
   `

diff --git a/...servability_onboarding/public/application/quickstart_flows/kubernetes/command_snippet.tsx b/...servability_onboarding/public/application/quickstart_flows/kubernetes/command_snippet.tsx
@@ -9,29 +9,30 @@ import React from 'react';
 import { EuiCodeBlock, EuiLink, EuiSpacer, EuiText } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { i18n } from '@kbn/i18n';
+import { ElasticAgentVersionInfo } from '../../../../common/types';
 import { buildKubectlCommand } from './build_kubectl_command';
 import { CopyToClipboardButton } from '../shared/copy_to_clipboard_button';
 
 interface Props {
   encodedApiKey: string;
   onboardingId: string;
   elasticsearchUrl: string;
-  elasticAgentVersion: string;
+  elasticAgentVersionInfo: ElasticAgentVersionInfo;
   isCopyPrimaryAction: boolean;
 }
 
 export function CommandSnippet({
   encodedApiKey,
   onboardingId,
   elasticsearchUrl,
-  elasticAgentVersion,
+  elasticAgentVersionInfo,
   isCopyPrimaryAction,
 }: Props) {
   const command = buildKubectlCommand({
     encodedApiKey,
     onboardingId,
     elasticsearchUrl,
-    elasticAgentVersion,
+    elasticAgentVersionInfo,
   });
 
   return (

diff --git a/...olution/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx b/...olution/observability_onboarding/public/application/quickstart_flows/kubernetes/index.tsx
@@ -58,7 +58,7 @@ export const KubernetesPanel: React.FC = () => {
               encodedApiKey={data.apiKeyEncoded}
               onboardingId={data.onboardingId}
               elasticsearchUrl={data.elasticsearchUrl}
-              elasticAgentVersion={data.elasticAgentVersion}
+              elasticAgentVersionInfo={data.elasticAgentVersionInfo}
               isCopyPrimaryAction={!isMonitoringStepActive}
             />
           )}

diff --git a/...solution/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx b/...solution/observability_onboarding/public/application/quickstart_flows/otel_logs/index.tsx
@@ -69,7 +69,9 @@ export const OtelLogsPanel: React.FC = () => {
   } = useKibana<ObservabilityOnboardingAppServices>();
 
   const AGENT_CDN_BASE_URL = 'artifacts.elastic.co/downloads/beats/elastic-agent';
-  const agentVersion = isServerless ? setup?.elasticAgentVersion : stackVersion;
+  const agentVersion =
+    isServerless && setup ? setup.elasticAgentVersionInfo.agentVersion : stackVersion;
+  const urlEncodedAgentVersion = encodeURIComponent(agentVersion);
 
   const allDatasetsLocator =
     share.url.locators.get<AllDatasetsLocatorParams>(ALL_DATASETS_LOCATOR_ID);
@@ -96,7 +98,7 @@ export const OtelLogsPanel: React.FC = () => {
       firstStepTitle: HOST_COMMAND,
       content: `arch=$(if ([[ $(arch) == "arm" || $(arch) == "aarch64" ]]); then echo "arm64"; else echo $(arch); fi)
 
-curl --output elastic-distro-${agentVersion}-linux-$arch.tar.gz --url https://${AGENT_CDN_BASE_URL}/elastic-agent-${agentVersion}-linux-$arch.tar.gz --proto '=https' --tlsv1.2 -fOL && mkdir -p elastic-distro-${agentVersion}-linux-$arch && tar -xvf elastic-distro-${agentVersion}-linux-$arch.tar.gz -C "elastic-distro-${agentVersion}-linux-$arch" --strip-components=1 && cd elastic-distro-${agentVersion}-linux-$arch
+curl --output elastic-distro-${agentVersion}-linux-$arch.tar.gz --url https://${AGENT_CDN_BASE_URL}/elastic-agent-${urlEncodedAgentVersion}-linux-$arch.tar.gz --proto '=https' --tlsv1.2 -fOL && mkdir -p elastic-distro-${agentVersion}-linux-$arch && tar -xvf elastic-distro-${agentVersion}-linux-$arch.tar.gz -C "elastic-distro-${agentVersion}-linux-$arch" --strip-components=1 && cd elastic-distro-${agentVersion}-linux-$arch
 
 rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mkdir -p ./data/otelcol && sed -i 's#\\\${env:STORAGE_DIR}#'"$PWD"/data/otelcol'#g' ./otel.yml && sed -i 's#\\\${env:ELASTIC_ENDPOINT}#${setup?.elasticsearchUrl}#g' ./otel.yml && sed -i 's/\\\${env:ELASTIC_API_KEY}/${apiKeyData?.apiKeyEncoded}/g' ./otel.yml`,
       start: 'sudo ./otelcol --config otel.yml',
@@ -108,7 +110,7 @@ rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mk
       firstStepTitle: HOST_COMMAND,
       content: `arch=$(if [[ $(uname -m) == "arm64" ]]; then echo "aarch64"; else echo $(uname -m); fi)
 
-curl --output elastic-distro-${agentVersion}-darwin-$arch.tar.gz --url https://${AGENT_CDN_BASE_URL}/elastic-agent-${agentVersion}-darwin-$arch.tar.gz --proto '=https' --tlsv1.2 -fOL && mkdir -p "elastic-distro-${agentVersion}-darwin-$arch" && tar -xvf elastic-distro-${agentVersion}-darwin-$arch.tar.gz -C "elastic-distro-${agentVersion}-darwin-$arch" --strip-components=1 && cd elastic-distro-${agentVersion}-darwin-$arch
+curl --output elastic-distro-${agentVersion}-darwin-$arch.tar.gz --url https://${AGENT_CDN_BASE_URL}/elastic-agent-${urlEncodedAgentVersion}-darwin-$arch.tar.gz --proto '=https' --tlsv1.2 -fOL && mkdir -p "elastic-distro-${agentVersion}-darwin-$arch" && tar -xvf elastic-distro-${agentVersion}-darwin-$arch.tar.gz -C "elastic-distro-${agentVersion}-darwin-$arch" --strip-components=1 && cd elastic-distro-${agentVersion}-darwin-$arch
 
 rm ./otel.yml && cp ./otel_samples/platformlogs_hostmetrics.yml ./otel.yml && mkdir -p ./data/otelcol  && sed -i '' 's#\\\${env:STORAGE_DIR}#'"$PWD"/data/otelcol'#g' ./otel.yml && sed -i '' 's#\\\${env:ELASTIC_ENDPOINT}#${setup?.elasticsearchUrl}#g' ./otel.yml && sed -i '' 's/\\\${env:ELASTIC_API_KEY}/${apiKeyData?.apiKeyEncoded}/g' ./otel.yml`,
       start: './otelcol --config otel.yml',

diff --git a/...k/plugins/observability_solution/observability_onboarding/server/lib/get_agent_version.ts b/...k/plugins/observability_solution/observability_onboarding/server/lib/get_agent_version.ts
@@ -6,14 +6,27 @@
  */
 
 import type { FleetStartContract } from '@kbn/fleet-plugin/server';
+import { ElasticAgentVersionInfo } from '../../common/types';
 
-export function getAgentVersion(fleetStart: FleetStartContract, kibanaVersion: string) {
+export async function getAgentVersionInfo(
+  fleetStart: FleetStartContract,
+  kibanaVersion: string
+): Promise<ElasticAgentVersionInfo> {
   // If undefined, we will follow fleet's strategy to select latest available version:
   // for serverless we will use the latest published version, for statefull we will use
   // current Kibana version. If false, irrespective of fleet flags and logic, we are
   // explicitly deciding to not append the current version.
   const includeCurrentVersion = kibanaVersion.endsWith('-SNAPSHOT') ? false : undefined;
 
   const agentClient = fleetStart.agentService.asInternalUser;
-  return agentClient.getLatestAgentAvailableVersion(includeCurrentVersion);
+  const [agentVersion, agentBaseVersion, agentDockerImageVersion] = await Promise.all([
+    agentClient.getLatestAgentAvailableVersion(includeCurrentVersion),
+    agentClient.getLatestAgentAvailableBaseVersion(includeCurrentVersion),
+    agentClient.getLatestAgentAvailableDockerImageVersion(includeCurrentVersion),
+  ]);
+  return {
+    agentVersion,
+    agentBaseVersion,
+    agentDockerImageVersion,
+  };
 }
diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts
@@ -20,7 +20,7 @@ import { ObservabilityOnboardingFlow } from '../../saved_objects/observability_o
 import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
 import { getHasLogs } from './get_has_logs';
 import { getKibanaUrl } from '../../lib/get_fallback_urls';
-import { getAgentVersion } from '../../lib/get_agent_version';
+import { getAgentVersionInfo } from '../../lib/get_agent_version';
 import { getFallbackESUrl } from '../../lib/get_fallback_urls';
 import { ElasticAgentStepPayload, InstalledIntegration, StepProgressPayloadRT } from '../types';
 import { createShipperApiKey } from '../../lib/api_key/create_shipper_api_key';
@@ -220,21 +220,22 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
 
     const fleetPluginStart = await plugins.fleet.start();
 
-    const [onboardingFlow, ingestApiKey, installApiKey, elasticAgentVersion] = await Promise.all([
-      saveObservabilityOnboardingFlow({
-        savedObjectsClient,
-        observabilityOnboardingState: {
-          type: 'autoDetect',
-          state: undefined,
-          progress: {},
-        },
-      }),
-      createShipperApiKey(client.asCurrentUser, `onboarding_ingest_${name}`),
-      (
-        await context.resolve(['core'])
-      ).core.security.authc.apiKeys.create(createInstallApiKey(`onboarding_install_${name}`)),
-      getAgentVersion(fleetPluginStart, kibanaVersion),
-    ]);
+    const [onboardingFlow, ingestApiKey, installApiKey, elasticAgentVersionInfo] =
+      await Promise.all([
+        saveObservabilityOnboardingFlow({
+          savedObjectsClient,
+          observabilityOnboardingState: {
+            type: 'autoDetect',
+            state: undefined,
+            progress: {},
+          },
+        }),
+        createShipperApiKey(client.asCurrentUser, `onboarding_ingest_${name}`),
+        (
+          await context.resolve(['core'])
+        ).core.security.authc.apiKeys.create(createInstallApiKey(`onboarding_install_${name}`)),
+        getAgentVersionInfo(fleetPluginStart, kibanaVersion),
+      ]);
 
     if (!installApiKey) {
       throw Boom.notFound('License does not allow API key creation.');
@@ -250,7 +251,7 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
       onboardingFlow,
       ingestApiKey: ingestApiKey.encoded,
       installApiKey: installApiKey.encoded,
-      elasticAgentVersion,
+      elasticAgentVersionInfo,
       kibanaUrl,
       scriptDownloadUrl,
     };

diff --git a/...plugins/observability_solution/observability_onboarding/server/routes/kubernetes/route.ts b/...plugins/observability_solution/observability_onboarding/server/routes/kubernetes/route.ts
@@ -10,17 +10,18 @@ import * as t from 'io-ts';
 import Boom from '@hapi/boom';
 import { termQuery } from '@kbn/observability-plugin/server';
 import type { estypes } from '@elastic/elasticsearch';
+import { ElasticAgentVersionInfo } from '../../../common/types';
 import { getFallbackESUrl } from '../../lib/get_fallback_urls';
 import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
 import { hasLogMonitoringPrivileges } from '../../lib/api_key/has_log_monitoring_privileges';
 import { createShipperApiKey } from '../../lib/api_key/create_shipper_api_key';
-import { getAgentVersion } from '../../lib/get_agent_version';
+import { getAgentVersionInfo } from '../../lib/get_agent_version';
 
 export interface CreateKubernetesOnboardingFlowRouteResponse {
   apiKeyEncoded: string;
   onboardingId: string;
   elasticsearchUrl: string;
-  elasticAgentVersion: string;
+  elasticAgentVersionInfo: ElasticAgentVersionInfo;
 }
 
 export interface HasKubernetesDataRouteResponse {
@@ -56,9 +57,9 @@ const createKubernetesOnboardingFlowRoute = createObservabilityOnboardingServerR
     const fleetPluginStart = await plugins.fleet.start();
     const packageClient = fleetPluginStart.packageService.asScoped(request);
 
-    const [{ encoded: apiKeyEncoded }, elasticAgentVersion] = await Promise.all([
+    const [{ encoded: apiKeyEncoded }, elasticAgentVersionInfo] = await Promise.all([
       createShipperApiKey(client.asCurrentUser, `${params.body.pkgName}_onboarding`, true),
-      getAgentVersion(fleetPluginStart, kibanaVersion),
+      getAgentVersionInfo(fleetPluginStart, kibanaVersion),
       // System package is always required
       packageClient.ensureInstalledPackage({ pkgName: 'system' }),
       // Kubernetes package is required for both classic kubernetes and otel
@@ -77,7 +78,7 @@ const createKubernetesOnboardingFlowRoute = createObservabilityOnboardingServerR
       onboardingId: uuidv4(),
       apiKeyEncoded,
       elasticsearchUrl: elasticsearchUrlList.length > 0 ? elasticsearchUrlList[0] : '',
-      elasticAgentVersion,
+      elasticAgentVersionInfo,
     };
   },
 });

diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/route.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/logs/route.ts
@@ -7,10 +7,11 @@
 
 import * as t from 'io-ts';
 import Boom from '@hapi/boom';
+import { ElasticAgentVersionInfo } from '../../../common/types';
 import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
 import { getFallbackESUrl } from '../../lib/get_fallback_urls';
 import { getKibanaUrl } from '../../lib/get_fallback_urls';
-import { getAgentVersion } from '../../lib/get_agent_version';
+import { getAgentVersionInfo } from '../../lib/get_agent_version';
 import { saveObservabilityOnboardingFlow } from '../../lib/state';
 import { createShipperApiKey } from '../../lib/api_key/create_shipper_api_key';
 import { ObservabilityOnboardingFlow } from '../../saved_objects/observability_onboarding_status';
@@ -39,7 +40,7 @@ const installShipperSetupRoute = createObservabilityOnboardingServerRoute({
   async handler(resources): Promise<{
     apiEndpoint: string;
     scriptDownloadUrl: string;
-    elasticAgentVersion: string;
+    elasticAgentVersionInfo: ElasticAgentVersionInfo;
     elasticsearchUrl: string[];
   }> {
     const {
@@ -50,7 +51,7 @@ const installShipperSetupRoute = createObservabilityOnboardingServerRoute({
     } = resources;
 
     const fleetPluginStart = await plugins.fleet.start();
-    const elasticAgentVersion = await getAgentVersion(fleetPluginStart, kibanaVersion);
+    const elasticAgentVersionInfo = await getAgentVersionInfo(fleetPluginStart, kibanaVersion);
     const kibanaUrl = getKibanaUrl(core.setup, plugins.cloud?.setup);
     const scriptDownloadUrl = new URL(
       core.setup.http.staticAssets.getPluginAssetHref('standalone_agent_setup.sh'),
@@ -67,7 +68,7 @@ const installShipperSetupRoute = createObservabilityOnboardingServerRoute({
       apiEndpoint,
       elasticsearchUrl,
       scriptDownloadUrl,
-      elasticAgentVersion,
+      elasticAgentVersionInfo,
     };
   },
 });