[Spacetime][Fleet] Introduce airgapped config for bundled packages (e…

…lastic#202435) Closes elastic#136617 Closes elastic#167195 ## Summary [Spacetime] Improving Integrations experience on airgapped envs using the existing `xpack.fleet.isAirGapped` configuration key: - Loading integrations is now much faster and doesn't attempt to contact the registry at all - Installing an uninstalled bundled packages should now be possible NOTE: Setting the `isAirGapped` skips the calls to registry altogether ### Testing - In `kibana.dev.yml`: - Make sure that APM and other bundled packages are not present in preconfiguration - Configure `xpack.fleet.registryUrl` to an unreacheable host, i.e. `xpack.fleet.registryUrl: http://notworking` - Configure `xpack.fleet.isAirGapped: true` - Copy the zip files from the EPR for some common bundled packages to them in `them in kibana/x-pack/plugins/fleet/target/bundled_packages` - Navigate to Integrations and verify that the page loads faster than in the past (it would take long because of retries) - Navigate to apm page or to`app/integrations/detail/apm-8.4.2/overview` - Try to install it it should succeed correctly ### Checklist - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: Elastic Machine <[email protected]>
gsoldevila · Dec 11, 2024 · c3f96bb · c3f96bb
1 parent 6220c57
commit c3f96bb
Show file tree

Hide file tree

Showing 13 changed files with 391 additions and 152 deletions.
diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts
@@ -125,6 +125,8 @@ export class SigningServiceNotFoundError extends FleetNotFoundError {}
 export class InputNotFoundError extends FleetNotFoundError {}
 export class OutputNotFoundError extends FleetNotFoundError {}
 export class PackageNotFoundError extends FleetNotFoundError {}
+export class ArchiveNotFoundError extends FleetNotFoundError {}
+
 export class PackagePolicyNotFoundError extends FleetNotFoundError<{
   /** The package policy ID that was not found */
   packagePolicyId: string;

diff --git a/x-pack/plugins/fleet/server/routes/epm/file_handler.ts b/x-pack/plugins/fleet/server/routes/epm/file_handler.ts
@@ -105,10 +105,17 @@ export const getFileHandler: FleetRequestHandler<
     });
   } else {
     const registryResponse = await getFile(pkgName, pkgVersion, filePath);
+    if (!registryResponse)
+      return response.custom({
+        body: {},
+        statusCode: 400,
+      });
+
     const headersToProxy: KnownHeaders[] = ['content-type'];
     const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => {
-      const value = registryResponse.headers.get(knownHeader);
-      if (value !== null) {
+      const value = registryResponse?.headers.get(knownHeader);
+
+      if (!!value) {
         headers[knownHeader] = value;
       }
       return headers;

diff --git a/x-pack/plugins/fleet/server/services/epm/airgapped.ts b/x-pack/plugins/fleet/server/services/epm/airgapped.ts
@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { appContextService } from '..';
+
+export const airGappedUtils = () => {
+  const config = appContextService.getConfig();
+  const hasRegistryUrls = config?.registryUrl || config?.registryProxyUrl;
+  const isAirGapped = config?.isAirGapped;
+
+  const shouldSkipRegistryRequests = isAirGapped && !hasRegistryUrls;
+
+  return { hasRegistryUrls, isAirGapped, shouldSkipRegistryRequests };
+};
diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts
@@ -925,6 +925,32 @@ owner: elastic`,
       });
     });
 
+    it('should avoid loading archive when isAirGapped == true', async () => {
+      const mockContract = createAppContextStartContractMock({ isAirGapped: true });
+      appContextService.start(mockContract);
+
+      const soClient = savedObjectsClientMock.create();
+      soClient.get.mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError());
+      MockRegistry.fetchInfo.mockResolvedValue({
+        name: 'my-package',
+        version: '1.0.0',
+        assets: [],
+      } as unknown as RegistryPackage);
+
+      await expect(
+        getPackageInfo({
+          savedObjectsClient: soClient,
+          pkgName: 'my-package',
+          pkgVersion: '1.0.0',
+        })
+      ).resolves.toMatchObject({
+        latestVersion: '1.0.0',
+        status: 'not_installed',
+      });
+
+      expect(MockRegistry.getPackage).not.toHaveBeenCalled();
+    });
+
     describe('installation status', () => {
       it('should be not_installed when no package SO exists', async () => {
         const soClient = savedObjectsClientMock.create();

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts
@@ -68,6 +68,8 @@ import { auditLoggingService } from '../../audit_logging';
 
 import { getFilteredSearchPackages } from '../filtered_packages';
 
+import { airGappedUtils } from '../airgapped';
+
 import { createInstallableFrom } from '.';
 import {
   getPackageAssetsMapCache,
@@ -478,7 +480,11 @@ export async function getPackageInfo({
   let packageInfo;
   // We need to get input only packages from source to get all fields
   // see https://github.com/elastic/package-registry/issues/864
-  if (registryInfo && skipArchive && registryInfo.type !== 'input') {
+  if (
+    registryInfo &&
+    (skipArchive || airGappedUtils().shouldSkipRegistryRequests) &&
+    registryInfo.type !== 'input'
+  ) {
     packageInfo = registryInfo;
     // Fix the paths
     paths =
@@ -629,6 +635,7 @@ export async function getPackageFromSource(options: {
     } catch (err) {
       if (err instanceof RegistryResponseError && err.status === 404) {
         res = await Registry.getBundledArchive(pkgName, pkgVersion);
+        logger.debug(`retrieved bundled package ${pkgName}-${pkgVersion}`);
       } else {
         throw err;
       }
@@ -763,7 +770,7 @@ export async function getPackageAssetsMap({
   packageInfo: PackageInfo;
   logger: Logger;
   ignoreUnverified?: boolean;
-}) {
+}): Promise<AssetsMap> {
   const cache = getPackageAssetsMapCache(packageInfo.name, packageInfo.version);
   if (cache) {
     return cache;
@@ -774,17 +781,22 @@ export async function getPackageAssetsMap({
     logger,
   });
 
-  let assetsMap: AssetsMap | undefined;
-  if (installedPackageWithAssets?.installation.version !== packageInfo.version) {
-    // Try to get from registry
-    const pkg = await Registry.getPackage(packageInfo.name, packageInfo.version, {
-      ignoreUnverified,
-    });
-    assetsMap = pkg.assetsMap;
-  } else {
-    assetsMap = installedPackageWithAssets.assetsMap;
-  }
-  setPackageAssetsMapCache(packageInfo.name, packageInfo.version, assetsMap);
+  try {
+    let assetsMap: AssetsMap | undefined;
+    if (installedPackageWithAssets?.installation.version !== packageInfo.version) {
+      // Try to get from registry
+      const pkg = await Registry.getPackage(packageInfo.name, packageInfo.version, {
+        ignoreUnverified,
+      });
+      assetsMap = pkg.assetsMap;
+    } else {
+      assetsMap = installedPackageWithAssets.assetsMap;
+    }
+    setPackageAssetsMapCache(packageInfo.name, packageInfo.version, assetsMap);
 
-  return assetsMap;
+    return assetsMap;
+  } catch (error) {
+    logger.warn(`getPackageAssetsMap error: ${error}`);
+    throw error;
+  }
 }
diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts
@@ -1354,44 +1354,48 @@ export async function installAssetsForInputPackagePolicy(opts: {
       `Error while creating index templates: unable to find installed package ${pkgInfo.name}`
     );
   }
-  if (installedPkgWithAssets.installation.version !== pkgInfo.version) {
-    const pkg = await Registry.getPackage(pkgInfo.name, pkgInfo.version, {
-      ignoreUnverified: force,
-    });
+  try {
+    if (installedPkgWithAssets.installation.version !== pkgInfo.version) {
+      const pkg = await Registry.getPackage(pkgInfo.name, pkgInfo.version, {
+        ignoreUnverified: force,
+      });
 
-    const archiveIterator = createArchiveIteratorFromMap(pkg.assetsMap);
-    packageInstallContext = {
-      assetsMap: pkg.assetsMap,
-      packageInfo: pkg.packageInfo,
-      paths: pkg.paths,
-      archiveIterator,
-    };
-  } else {
-    const archiveIterator = createArchiveIteratorFromMap(installedPkgWithAssets.assetsMap);
-    packageInstallContext = {
-      assetsMap: installedPkgWithAssets.assetsMap,
-      packageInfo: installedPkgWithAssets.packageInfo,
-      paths: installedPkgWithAssets.paths,
-      archiveIterator,
-    };
-  }
+      const archiveIterator = createArchiveIteratorFromMap(pkg.assetsMap);
+      packageInstallContext = {
+        assetsMap: pkg.assetsMap,
+        packageInfo: pkg.packageInfo,
+        paths: pkg.paths,
+        archiveIterator,
+      };
+    } else {
+      const archiveIterator = createArchiveIteratorFromMap(installedPkgWithAssets.assetsMap);
+      packageInstallContext = {
+        assetsMap: installedPkgWithAssets.assetsMap,
+        packageInfo: installedPkgWithAssets.packageInfo,
+        paths: installedPkgWithAssets.paths,
+        archiveIterator,
+      };
+    }
 
-  await installIndexTemplatesAndPipelines({
-    installedPkg: installedPkgWithAssets.installation,
-    packageInstallContext,
-    esReferences: installedPkgWithAssets.installation.installed_es || [],
-    savedObjectsClient: soClient,
-    esClient,
-    logger,
-    onlyForDataStreams: [dataStream],
-  });
-  // Upate ES index patterns
-  await optimisticallyAddEsAssetReferences(
-    soClient,
-    installedPkgWithAssets.installation.name,
-    [],
-    generateESIndexPatterns([dataStream])
-  );
+    await installIndexTemplatesAndPipelines({
+      installedPkg: installedPkgWithAssets.installation,
+      packageInstallContext,
+      esReferences: installedPkgWithAssets.installation.installed_es || [],
+      savedObjectsClient: soClient,
+      esClient,
+      logger,
+      onlyForDataStreams: [dataStream],
+    });
+    // Upate ES index patterns
+    await optimisticallyAddEsAssetReferences(
+      soClient,
+      installedPkgWithAssets.installation.name,
+      [],
+      generateESIndexPatterns([dataStream])
+    );
+  } catch (error) {
+    logger.warn(`installAssetsForInputPackagePolicy error: ${error}`);
+  }
 }
 
 interface NoPkgArgs {

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts
@@ -75,7 +75,7 @@ export async function verifyPackageArchiveSignature({
 }: {
   pkgName: string;
   pkgVersion: string;
-  pkgArchiveBuffer: Buffer;
+  pkgArchiveBuffer: Buffer | undefined;
   logger: Logger;
 }): Promise<PackageVerificationResult> {
   const verificationKey = await getGpgKeyOrUndefined();
@@ -97,6 +97,11 @@ export async function verifyPackageArchiveSignature({
     return result;
   }
 
+  if (!pkgArchiveBuffer) {
+    logger.warn(`Archive not found for package ${pkgName}-${pkgVersion}. Skipping verification.`);
+    return result;
+  }
+
   const { isVerified, keyId } = await _verifyPackageSignature({
     pkgArchiveBuffer,
     pkgArchiveSignature,

diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts
@@ -84,6 +84,7 @@ describe('splitPkgKey', () => {
 describe('fetch package', () => {
   afterEach(() => {
     mockFetchUrl.mockReset();
+    mockGetConfig.mockReset();
     mockGetBundledPackageByName.mockReset();
   });
 
@@ -100,6 +101,22 @@ describe('fetch package', () => {
       expect(result).toEqual(registryPackage);
     });
 
+    it('Should return bundled package when isAirGapped = true', async () => {
+      mockGetConfig.mockReturnValue({
+        isAirGapped: true,
+        enabled: true,
+        agents: { enabled: true, elasticsearch: {} },
+      });
+      const bundledPackage = { name: 'testpkg', version: '1.0.0' };
+      const registryPackage = { name: 'testpkg', version: '1.0.1' };
+
+      mockFetchUrl.mockResolvedValue(JSON.stringify([registryPackage]));
+
+      mockGetBundledPackageByName.mockResolvedValue(bundledPackage);
+      const result = await fetchMethodToTest('testpkg');
+      expect(result).toEqual(bundledPackage);
+    });
+
     it('Should return bundled package if bundled package is newer version', async () => {
       const bundledPackage = { name: 'testpkg', version: '1.0.1' };
       const registryPackage = { name: 'testpkg', version: '1.0.0' };
@@ -220,6 +237,15 @@ describe('fetchInfo', () => {
       expect(e).toBeInstanceOf(PackageNotFoundError);
     }
   });
+
+  it('falls back to bundled package when isAirGapped config == true', async () => {
+    mockGetConfig.mockReturnValue({
+      isAirGapped: true,
+    });
+
+    const fetchedInfo = await fetchInfo('test-package', '1.0.0');
+    expect(fetchedInfo).toBeTruthy();
+  });
 });
 
 describe('fetchCategories', () => {
@@ -317,6 +343,13 @@ describe('fetchList', () => {
     expect(callUrl.searchParams.get('capabilities')).toBeNull();
   });
 
+  it('does not call registry if isAirGapped == true', async () => {
+    mockGetConfig.mockReturnValue({ isAirGapped: true });
+    mockFetchUrl.mockResolvedValue(JSON.stringify([]));
+    await fetchList();
+    expect(mockFetchUrl).toBeCalledTimes(0);
+  });
+
   it('does call registry with kibana.version if not explictly disabled', async () => {
     mockGetConfig.mockReturnValue({
       internal: {