[Fleet] Fix inability to upgrade agents from 8.10.4 -> 8.11

x-pack/plugins/fleet/server/routes/agent/handlers.ts

@@ -354,8 +354,9 @@ function isStringArray(arr: unknown | string[]): arr is string[] {
 
 export const getAvailableVersionsHandler: RequestHandler = async (context, request, response) => {
   try {
-    const availableVersions = await AgentService.getAvailableVersions({});
+    const availableVersions = await AgentService.getAvailableVersions();
     const body: GetAvailableVersionsResponse = { items: availableVersions };
+
     return response.ok({ body });
   } catch (error) {
     return defaultFleetErrorHandler({ error, response });

x-pack/plugins/fleet/server/services/agents/crud.test.ts

@@ -9,7 +9,9 @@ import type { ElasticsearchClient } from '@kbn/core/server';
 import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks';
 
 import { AGENTS_INDEX } from '../../constants';
+import { createAppContextStartContractMock } from '../../mocks';
 import type { Agent } from '../../types';
+import { appContextService } from '../app_context';
 
 import { auditLoggingService } from '../audit_logging';
 
@@ -30,6 +32,7 @@ const mockedAuditLoggingService = auditLoggingService as jest.Mocked<typeof audi
 
 describe('Agents CRUD test', () => {
   const soClientMock = savedObjectsClientMock.create();
+  let mockContract: ReturnType<typeof createAppContextStartContractMock>;
   let esClientMock: ElasticsearchClient;
   let searchMock: jest.Mock;
 
@@ -41,6 +44,9 @@ describe('Agents CRUD test', () => {
       openPointInTime: jest.fn().mockResolvedValue({ id: '1' }),
       closePointInTime: jest.fn(),
     } as unknown as ElasticsearchClient;
+
+    mockContract = createAppContextStartContractMock();
+    appContextService.start(mockContract);
   });
 
   function getEsResponse(ids: string[], total: number) {

x-pack/plugins/fleet/server/services/agents/versions.test.ts

@@ -7,6 +7,8 @@
 
 import { readFile } from 'fs/promises';
 
+import fetch from 'node-fetch';
+
 let mockKibanaVersion = '300.0.0';
 let mockConfig = {};
 jest.mock('../app_context', () => {
@@ -21,25 +23,40 @@ jest.mock('../app_context', () => {
 });
 
 jest.mock('fs/promises');
+jest.mock('node-fetch');
 
 const mockedReadFile = readFile as jest.MockedFunction<typeof readFile>;
+const mockedFetch = fetch as jest.MockedFunction<typeof fetch>;
+
+const emptyResponse = {
+  status: 200,
+  text: jest.fn().mockResolvedValue(JSON.stringify({})),
+} as any;
+
 import { getAvailableVersions } from './versions';
 
 describe('getAvailableVersions', () => {
+  beforeEach(() => {
+    mockedReadFile.mockReset();
+    mockedFetch.mockReset();
+  });
+
   it('should return available version and filter version < 7.17', async () => {
     mockKibanaVersion = '300.0.0';
     mockedReadFile.mockResolvedValue(`["8.1.0", "8.0.0", "7.17.0", "7.16.0"]`);
+    mockedFetch.mockResolvedValueOnce(emptyResponse);
 
-    const res = await getAvailableVersions({ cached: false, includeCurrentVersion: true });
+    const res = await getAvailableVersions({ includeCurrentVersion: true, ignoreCache: true });
 
     expect(res).toEqual(['300.0.0', '8.1.0', '8.0.0', '7.17.0']);
   });
 
   it('should not strip -SNAPSHOT from kibana version', async () => {
     mockKibanaVersion = '300.0.0-SNAPSHOT';
     mockedReadFile.mockResolvedValue(`["8.1.0", "8.0.0", "7.17.0", "7.16.0"]`);
+    mockedFetch.mockResolvedValueOnce(emptyResponse);
 
-    const res = await getAvailableVersions({ cached: false, includeCurrentVersion: true });
+    const res = await getAvailableVersions({ includeCurrentVersion: true, ignoreCache: true });
     expect(res).toEqual(['300.0.0-SNAPSHOT', '8.1.0', '8.0.0', '7.17.0']);
   });
 
@@ -51,17 +68,19 @@ describe('getAvailableVersions', () => {
       },
     };
     mockedReadFile.mockResolvedValue(`["8.1.0", "8.0.0", "7.17.0", "7.16.0"]`);
+    mockedFetch.mockResolvedValueOnce(emptyResponse);
 
-    const res = await getAvailableVersions({ cached: false });
+    const res = await getAvailableVersions({ ignoreCache: true });
 
     expect(res).toEqual(['8.1.0', '8.0.0', '7.17.0']);
   });
 
   it('should not include the current version if includeCurrentVersion = false', async () => {
     mockKibanaVersion = '300.0.0-SNAPSHOT';
     mockedReadFile.mockResolvedValue(`["8.1.0", "8.0.0", "7.17.0", "7.16.0"]`);
+    mockedFetch.mockResolvedValueOnce(emptyResponse);
 
-    const res = await getAvailableVersions({ cached: false, includeCurrentVersion: false });
+    const res = await getAvailableVersions({ includeCurrentVersion: false, ignoreCache: true });
 
     expect(res).toEqual(['8.1.0', '8.0.0', '7.17.0']);
   });
@@ -74,9 +93,90 @@ describe('getAvailableVersions', () => {
       },
     };
     mockedReadFile.mockRejectedValue({ code: 'ENOENT' });
+    mockedFetch.mockResolvedValueOnce(emptyResponse);
 
-    const res = await getAvailableVersions({ cached: false });
+    const res = await getAvailableVersions({ ignoreCache: true });
 
     expect(res).toEqual(['300.0.0']);
   });
+
+  it('should include versions returned from product_versions API', async () => {
+    mockKibanaVersion = '300.0.0';
+    mockedReadFile.mockResolvedValue(`["8.1.0", "8.0.0", "7.17.0", "7.16.0"]`);
+    mockedFetch.mockResolvedValueOnce({
+      status: 200,
+      text: jest.fn().mockResolvedValue(
+        JSON.stringify([
+          [
+            {
+              title: 'Elastic Agent 8.1.0',
+              version_number: '8.1.0',
+            },
+            {
+              title: 'Elastic Agent 8.10.0',
+              version_number: '8.10.0',
+            },
+            {
+              title: 'Elastic Agent 8.9.2',
+              version_number: '8.9.2',
+            },
+            ,
+          ],
+        ])
+      ),
+    } as any);
+
+    const res = await getAvailableVersions({ ignoreCache: true });
+
+    // Should sort, uniquify and filter out versions < 7.17
+    expect(res).toEqual(['8.10.0', '8.9.2', '8.1.0', '8.0.0', '7.17.0']);
+  });
+
+  it('should cache results', async () => {
+    mockKibanaVersion = '300.0.0';
+    mockedReadFile.mockResolvedValue(`["8.1.0", "8.0.0", "7.17.0", "7.16.0"]`);
+    mockedFetch.mockResolvedValueOnce({
+      status: 200,
+      text: jest.fn().mockResolvedValue(
+        JSON.stringify([
+          [
+            {
+              title: 'Elastic Agent 8.1.0',
+              version_number: '8.1.0',
+            },
+            {
+              title: 'Elastic Agent 8.10.0',
+              version_number: '8.10.0',
+            },
+            {
+              title: 'Elastic Agent 8.9.2',
+              version_number: '8.9.2',
+            },
+            ,
+          ],
+        ])
+      ),
+    } as any);
+
+    await getAvailableVersions();
+
+    mockedFetch.mockResolvedValueOnce({
+      status: 200,
+      text: jest.fn().mockResolvedValue(
+        JSON.stringify([
+          [
+            {
+              title: 'Elastic Agent 300.0.0',
+              version_number: '300.0.0',
+            },
+          ],
+        ])
+      ),
+    } as any);
+
+    const res2 = await getAvailableVersions();
+
+    expect(mockedFetch).toBeCalledTimes(1);
+    expect(res2).not.toContain('300.0.0');
+  });
 });

x-pack/plugins/fleet/server/services/agents/versions.ts

@@ -8,18 +8,27 @@
 import { readFile } from 'fs/promises';
 import Path from 'path';
 
-import { REPO_ROOT } from '@kbn/repo-info';
+import fetch from 'node-fetch';
+import pRetry from 'p-retry';
 import { uniq } from 'lodash';
 import semverGte from 'semver/functions/gte';
 import semverGt from 'semver/functions/gt';
 import semverCoerce from 'semver/functions/coerce';
 
+import { REPO_ROOT } from '@kbn/repo-info';
+
 import { appContextService } from '..';
 
 const MINIMUM_SUPPORTED_VERSION = '7.17.0';
 const AGENT_VERSION_BUILD_FILE = 'x-pack/plugins/fleet/target/agent_versions_list.json';
 
-let availableVersions: string[] | undefined;
+// Endpoint maintained by the web-team and hosted on the elastic website
+const PRODUCT_VERSIONS_URL = 'https://www.elastic.co/api/product_versions';
+
+// Cache available versions in memory for 1 hour
+const CACHE_DURATION = 1000 * 60 * 60;
+let CACHED_AVAILABLE_VERSIONS: string[] | undefined;
+let LAST_FETCHED: number | undefined;
 
 export const getLatestAvailableVersion = async (
   includeCurrentVersion?: boolean
@@ -30,54 +39,100 @@ export const getLatestAvailableVersion = async (
 };
 
 export const getAvailableVersions = async ({
-  cached = true,
   includeCurrentVersion,
+  ignoreCache = false, // This is only here to allow us to ignore the cache in tests
 }: {
-  cached?: boolean;
   includeCurrentVersion?: boolean;
-}): Promise<string[]> => {
-  // Use cached value to avoid reading from disk each time
-  if (cached && availableVersions) {
-    return availableVersions;
+  ignoreCache?: boolean;
+} = {}): Promise<string[]> => {
+  const logger = appContextService.getLogger();
+
+  if (LAST_FETCHED && !ignoreCache) {
+    const msSinceLastFetched = Date.now() - (LAST_FETCHED || 0);
+
+    if (msSinceLastFetched < CACHE_DURATION && CACHED_AVAILABLE_VERSIONS !== undefined) {
+      logger.debug(`Cache is valid, returning cached available versions`);
+
+      return CACHED_AVAILABLE_VERSIONS;
+    }
+
+    logger.debug('Cache has expired, fetching available versions from disk + API');
   }
 
-  // Read a static file generated at build time
   const config = appContextService.getConfig();
-  let versionsToDisplay: string[] = [];
-
   const kibanaVersion = appContextService.getKibanaVersion();
 
+  let availableVersions: string[] = [];
+
+  // First, grab available versions from the static file that's placed on disk at build time
   try {
     const file = await readFile(Path.join(REPO_ROOT, AGENT_VERSION_BUILD_FILE), 'utf-8');
-
-    // Exclude versions older than MINIMUM_SUPPORTED_VERSION and pre-release versions (SNAPSHOT, rc..)
-    // De-dup and sort in descending order
     const data: string[] = JSON.parse(file);
 
-    const versions = data
-      .map((item: any) => semverCoerce(item)?.version || '')
-      .filter((v: any) => semverGte(v, MINIMUM_SUPPORTED_VERSION))
-      .sort((a: any, b: any) => (semverGt(a, b) ? -1 : 1));
-    versionsToDisplay = uniq(versions) as string[];
+    availableVersions = [...availableVersions, ...data];
+  } catch (error) {
+    // If we can't read from the file, the error is non-blocking. We'll try to source data from the
+    // product versions API later.
+    logger.debug(`Error reading file ${AGENT_VERSION_BUILD_FILE}: ${error.message}`);
+  }
+
+  // Next, fetch from the product versions API. This API call is aggressively cached, so we won't
+  // fetch from the live API more than `TIME_BETWEEN_FETCHES` milliseconds.
+  const apiVersions = await fetchAgentVersionsFromApi();
+
+  // Coerce each version to a semver object and compare to our `MINIMUM_SUPPORTED_VERSION` - we
+  // only want support versions in the final result. We'll also sort by newest version first.
+  availableVersions = uniq([...availableVersions, ...apiVersions])
+    .map((item: any) => semverCoerce(item)?.version || '')
+    .filter((v: any) => semverGte(v, MINIMUM_SUPPORTED_VERSION))
+    .sort((a: any, b: any) => (semverGt(a, b) ? -1 : 1));
+
+  // If the current stack version isn't included in the list of available versions, add it
+  // at the front of the array
+  const hasCurrentVersion = availableVersions.some((v) => v === kibanaVersion);
+  if (includeCurrentVersion && !hasCurrentVersion) {
+    availableVersions = [kibanaVersion, ...availableVersions];
+  }
 
-    const appendCurrentVersion = includeCurrentVersion;
+  // Allow upgrading to the current stack version if this override flag is provided via `kibana.yml`.
+  // This is useful for development purposes.
+  if (availableVersions.length === 0 && !config?.internal?.onlyAllowAgentUpgradeToKnownVersions) {
+    availableVersions = [kibanaVersion];
+  }
 
-    if (appendCurrentVersion) {
-      // Add current version if not already present
-      const hasCurrentVersion = versionsToDisplay.some((v) => v === kibanaVersion);
+  // Don't prime the cache in tests
+  if (!ignoreCache) {
+    CACHED_AVAILABLE_VERSIONS = availableVersions;
+    LAST_FETCHED = Date.now();
+  }
 
-      versionsToDisplay = !hasCurrentVersion
-        ? [kibanaVersion].concat(versionsToDisplay)
-        : versionsToDisplay;
-    }
+  return availableVersions;
+};
 
-    availableVersions = versionsToDisplay;
+async function fetchAgentVersionsFromApi() {
+  const logger = appContextService.getLogger();
 
-    return availableVersions;
-  } catch (e) {
-    if (e.code === 'ENOENT') {
-      return config?.internal?.onlyAllowAgentUpgradeToKnownVersions ? [] : [kibanaVersion];
-    }
-    throw e;
+  const options = {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  };
+
+  const response = await pRetry(() => fetch(PRODUCT_VERSIONS_URL, options), { retries: 1 });
+  const rawBody = await response.text();
+
+  // We need to handle non-200 responses gracefully here to support airgapped environments where
+  // Kibana doesn't have internet access to query this API
+  if (response.status >= 400) {
+    logger.debug(`Status code ${response.status} received from versions API: ${rawBody}`);
+    return [];
   }
-};
+
+  const jsonBody = JSON.parse(rawBody);
+
+  const versions: string[] = (jsonBody.length ? jsonBody[0] : [])
+    .filter((item: any) => item?.title?.includes('Elastic Agent'))
+    .map((item: any) => item?.version_number);
+
+  return versions;
+}