From 25e586acd0eac7207379872c59e4a40c9e9026d0 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Wed, 17 Mar 2021 10:16:18 +0100 Subject: [PATCH] Migrate the optimizer mixin to core (#94272) * migrate optimizer mixin to core apps * fix core_app tests * add integration tests, extract selectCompressedFile * add CoreApp unit test * more unit tests * unit tests for bundle_route * more unit tests * remove /src/optimize/ from codeowners * fix case * NIT --- .github/CODEOWNERS | 1 - .../bundle_routes/bundle_route.test.mocks.ts | 12 + .../bundle_routes/bundle_route.test.ts | 70 +++++ .../core_app/bundle_routes/bundles_route.ts | 49 +++ .../bundle_routes/dynamic_asset_response.ts | 124 ++++++++ .../bundle_routes/file_hash.test.mocks.ts | 15 + .../core_app/bundle_routes/file_hash.test.ts | 72 +++++ .../core_app/bundle_routes/file_hash.ts | 32 ++ .../bundle_routes/file_hash_cache.test.ts | 33 ++ .../bundle_routes}/file_hash_cache.ts | 18 +- src/core/server/core_app/bundle_routes/fs.ts | 17 + .../server/core_app/bundle_routes/index.ts} | 6 +- .../register_bundle_routes.test.mocks.ts | 16 + .../register_bundle_routes.test.ts | 101 ++++++ .../bundle_routes/register_bundle_routes.ts | 69 ++++ .../bundle_routes/select_compressed_file.ts | 56 ++++ .../server/core_app/bundle_routes/utils.ts} | 37 +-- .../server/core_app/core_app.test.mocks.ts} | 6 +- src/core/server/core_app/core_app.test.ts | 35 ++- src/core/server/core_app/core_app.ts | 20 +- .../__fixtures__/outside_output.js | 0 .../__fixtures__/plugin/foo/gzip_chunk.js} | 2 +- .../__fixtures__/plugin/foo/gzip_chunk.js.gz | Bin 0 -> 256 bytes .../__fixtures__/plugin/foo/image.png | Bin .../__fixtures__/plugin/foo/plugin.js | 0 .../integration_tests/bundle_routes.test.ts | 171 ++++++++++ src/core/server/server.ts | 2 +- src/legacy/server/kbn_server.js | 6 +- .../bundles_route/bundles_route.test.ts | 296 ------------------ src/optimize/bundles_route/bundles_route.ts | 131 -------- .../bundles_route/dynamic_asset_response.ts | 157 ---------- .../bundles_route/proxy_bundles_route.ts | 35 --- src/optimize/np_ui_plugin_public_dirs.ts | 40 --- src/optimize/optimize_mixin.ts | 28 -- 34 files changed, 917 insertions(+), 740 deletions(-) create mode 100644 src/core/server/core_app/bundle_routes/bundle_route.test.mocks.ts create mode 100644 src/core/server/core_app/bundle_routes/bundle_route.test.ts create mode 100644 src/core/server/core_app/bundle_routes/bundles_route.ts create mode 100644 src/core/server/core_app/bundle_routes/dynamic_asset_response.ts create mode 100644 src/core/server/core_app/bundle_routes/file_hash.test.mocks.ts create mode 100644 src/core/server/core_app/bundle_routes/file_hash.test.ts create mode 100644 src/core/server/core_app/bundle_routes/file_hash.ts create mode 100644 src/core/server/core_app/bundle_routes/file_hash_cache.test.ts rename src/{optimize/bundles_route => core/server/core_app/bundle_routes}/file_hash_cache.ts (59%) create mode 100644 src/core/server/core_app/bundle_routes/fs.ts rename src/{optimize/jest.config.js => core/server/core_app/bundle_routes/index.ts} (77%) create mode 100644 src/core/server/core_app/bundle_routes/register_bundle_routes.test.mocks.ts create mode 100644 src/core/server/core_app/bundle_routes/register_bundle_routes.test.ts create mode 100644 src/core/server/core_app/bundle_routes/register_bundle_routes.ts create mode 100644 src/core/server/core_app/bundle_routes/select_compressed_file.ts rename src/{optimize/bundles_route/file_hash.ts => core/server/core_app/bundle_routes/utils.ts} (51%) rename src/{optimize/bundles_route/index.ts => core/server/core_app/core_app.test.mocks.ts} (70%) rename src/{optimize/bundles_route => core/server/core_app/integration_tests}/__fixtures__/outside_output.js (100%) rename src/{optimize/index.ts => core/server/core_app/integration_tests/__fixtures__/plugin/foo/gzip_chunk.js} (87%) create mode 100644 src/core/server/core_app/integration_tests/__fixtures__/plugin/foo/gzip_chunk.js.gz rename src/{optimize/bundles_route => core/server/core_app/integration_tests}/__fixtures__/plugin/foo/image.png (100%) rename src/{optimize/bundles_route => core/server/core_app/integration_tests}/__fixtures__/plugin/foo/plugin.js (100%) create mode 100644 src/core/server/core_app/integration_tests/bundle_routes.test.ts delete mode 100644 src/optimize/bundles_route/bundles_route.test.ts delete mode 100644 src/optimize/bundles_route/bundles_route.ts delete mode 100644 src/optimize/bundles_route/dynamic_asset_response.ts delete mode 100644 src/optimize/bundles_route/proxy_bundles_route.ts delete mode 100644 src/optimize/np_ui_plugin_public_dirs.ts delete mode 100644 src/optimize/optimize_mixin.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4f8d3fa4dc429..508cd8f9e8007 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -145,7 +145,6 @@ # Operations /src/dev/ @elastic/kibana-operations /src/setup_node_env/ @elastic/kibana-operations -/src/optimize/ @elastic/kibana-operations /packages/*eslint*/ @elastic/kibana-operations /packages/*babel*/ @elastic/kibana-operations /packages/kbn-dev-utils*/ @elastic/kibana-operations diff --git a/src/core/server/core_app/bundle_routes/bundle_route.test.mocks.ts b/src/core/server/core_app/bundle_routes/bundle_route.test.mocks.ts new file mode 100644 index 0000000000000..c7839f6a26e8b --- /dev/null +++ b/src/core/server/core_app/bundle_routes/bundle_route.test.mocks.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const createDynamicAssetHandlerMock = jest.fn(); +jest.doMock('./dynamic_asset_response', () => ({ + createDynamicAssetHandler: createDynamicAssetHandlerMock, +})); diff --git a/src/core/server/core_app/bundle_routes/bundle_route.test.ts b/src/core/server/core_app/bundle_routes/bundle_route.test.ts new file mode 100644 index 0000000000000..377d8432ae9a9 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/bundle_route.test.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createDynamicAssetHandlerMock } from './bundle_route.test.mocks'; + +import { httpServiceMock } from '../../http/http_service.mock'; +import { FileHashCache } from './file_hash_cache'; +import { registerRouteForBundle } from './bundles_route'; + +describe('registerRouteForBundle', () => { + let router: ReturnType; + let fileHashCache: FileHashCache; + + beforeEach(() => { + router = httpServiceMock.createRouter(); + fileHashCache = new FileHashCache(); + }); + + afterEach(() => { + createDynamicAssetHandlerMock.mockReset(); + }); + + it('calls `router.get` with the correct parameters', () => { + const handler = jest.fn(); + createDynamicAssetHandlerMock.mockReturnValue(handler); + + registerRouteForBundle(router, { + isDist: false, + publicPath: '/public-path/', + bundlesPath: '/bundle-path', + fileHashCache, + routePath: '/route-path/', + }); + + expect(router.get).toHaveBeenCalledTimes(1); + expect(router.get).toHaveBeenCalledWith( + { + path: '/route-path/{path*}', + options: { + authRequired: false, + }, + validate: expect.any(Object), + }, + handler + ); + }); + + it('calls `createDynamicAssetHandler` with the correct parameters', () => { + registerRouteForBundle(router, { + isDist: false, + publicPath: '/public-path/', + bundlesPath: '/bundle-path', + fileHashCache, + routePath: '/route-path/', + }); + + expect(createDynamicAssetHandlerMock).toHaveBeenCalledTimes(1); + expect(createDynamicAssetHandlerMock).toHaveBeenCalledWith({ + isDist: false, + publicPath: '/public-path/', + bundlesPath: '/bundle-path', + fileHashCache, + }); + }); +}); diff --git a/src/core/server/core_app/bundle_routes/bundles_route.ts b/src/core/server/core_app/bundle_routes/bundles_route.ts new file mode 100644 index 0000000000000..c15babe13a2ce --- /dev/null +++ b/src/core/server/core_app/bundle_routes/bundles_route.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema } from '@kbn/config-schema'; +import { IRouter } from '../../http'; +import { createDynamicAssetHandler } from './dynamic_asset_response'; +import { FileHashCache } from './file_hash_cache'; + +export function registerRouteForBundle( + router: IRouter, + { + publicPath, + routePath, + bundlesPath, + fileHashCache, + isDist, + }: { + publicPath: string; + routePath: string; + bundlesPath: string; + fileHashCache: FileHashCache; + isDist: boolean; + } +) { + router.get( + { + path: `${routePath}{path*}`, + options: { + authRequired: false, + }, + validate: { + params: schema.object({ + path: schema.string(), + }), + }, + }, + createDynamicAssetHandler({ + publicPath, + bundlesPath, + isDist, + fileHashCache, + }) + ); +} diff --git a/src/core/server/core_app/bundle_routes/dynamic_asset_response.ts b/src/core/server/core_app/bundle_routes/dynamic_asset_response.ts new file mode 100644 index 0000000000000..1ad03608999c7 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/dynamic_asset_response.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createReadStream } from 'fs'; +import { resolve, extname } from 'path'; +import mime from 'mime-types'; +import agent from 'elastic-apm-node'; + +import { fstat, close } from './fs'; +import { RequestHandler } from '../../http'; +import { IFileHashCache } from './file_hash_cache'; +import { getFileHash } from './file_hash'; +import { selectCompressedFile } from './select_compressed_file'; + +const MINUTE = 60; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; + +/** + * Serve asset for the requested path. This is designed + * to replicate a subset of the features provided by Hapi's Inert + * plugin including: + * - ensure path is not traversing out of the bundle directory + * - manage use file descriptors for file access to efficiently + * interact with the file multiple times in each request + * - generate and cache etag for the file + * - write correct headers to response for client-side caching + * and invalidation + * - stream file to response + * + * It differs from Inert in some important ways: + * - cached hash/etag is based on the file on disk, but modified + * by the public path so that individual public paths have + * different etags, but can share a cache + */ +export const createDynamicAssetHandler = ({ + bundlesPath, + fileHashCache, + isDist, + publicPath, +}: { + bundlesPath: string; + publicPath: string; + fileHashCache: IFileHashCache; + isDist: boolean; +}): RequestHandler<{ path: string }, {}, {}> => { + return async (ctx, req, res) => { + agent.setTransactionName('GET ?/bundles/?'); + + let fd: number | undefined; + let fileEncoding: 'gzip' | 'br' | undefined; + + try { + const path = resolve(bundlesPath, req.params.path); + + // prevent path traversal, only process paths that resolve within bundlesPath + if (!path.startsWith(bundlesPath)) { + return res.forbidden({ + body: 'EACCES', + }); + } + + // we use and manage a file descriptor mostly because + // that's what Inert does, and since we are accessing + // the file 2 or 3 times per request it seems logical + ({ fd, fileEncoding } = await selectCompressedFile( + req.headers['accept-encoding'] as string, + path + )); + + let headers: Record; + if (isDist) { + headers = { 'cache-control': `max-age=${365 * DAY}` }; + } else { + const stat = await fstat(fd); + const hash = await getFileHash(fileHashCache, path, stat, fd); + headers = { + etag: `${hash}-${publicPath}`, + 'cache-control': 'must-revalidate', + }; + } + + // If we manually selected a compressed file, specify the encoding header. + // Otherwise, let Hapi automatically gzip the response. + if (fileEncoding) { + headers['content-encoding'] = fileEncoding; + } + + const fileExt = extname(path); + const contentType = mime.lookup(fileExt); + const mediaType = mime.contentType(contentType || fileExt); + headers['content-type'] = mediaType || ''; + + const content = createReadStream(null as any, { + fd, + start: 0, + autoClose: true, + }); + + return res.ok({ + body: content, + headers, + }); + } catch (error) { + if (fd) { + try { + await close(fd); + } catch (_) { + // ignore errors from close, we already have one to report + // and it's very likely they are the same + } + } + if (error.code === 'ENOENT') { + return res.notFound(); + } + throw error; + } + }; +}; diff --git a/src/core/server/core_app/bundle_routes/file_hash.test.mocks.ts b/src/core/server/core_app/bundle_routes/file_hash.test.mocks.ts new file mode 100644 index 0000000000000..d7f6812ba5d29 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/file_hash.test.mocks.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const generateFileHashMock = jest.fn(); +export const getFileCacheKeyMock = jest.fn(); + +jest.doMock('./utils', () => ({ + generateFileHash: generateFileHashMock, + getFileCacheKey: getFileCacheKeyMock, +})); diff --git a/src/core/server/core_app/bundle_routes/file_hash.test.ts b/src/core/server/core_app/bundle_routes/file_hash.test.ts new file mode 100644 index 0000000000000..918f435156344 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/file_hash.test.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { generateFileHashMock, getFileCacheKeyMock } from './file_hash.test.mocks'; + +import { resolve } from 'path'; +import { Stats } from 'fs'; +import { getFileHash } from './file_hash'; +import { IFileHashCache } from './file_hash_cache'; + +const mockedCache = (): jest.Mocked => ({ + del: jest.fn(), + get: jest.fn(), + set: jest.fn(), +}); + +describe('getFileHash', () => { + const sampleFilePath = resolve(__dirname, 'foo.js'); + const fd = 42; + const stats: Stats = { ino: 42, size: 9000 } as any; + + beforeEach(() => { + getFileCacheKeyMock.mockImplementation((path: string, stat: Stats) => `${path}-${stat.ino}`); + }); + + afterEach(() => { + generateFileHashMock.mockReset(); + getFileCacheKeyMock.mockReset(); + }); + + it('returns the value from cache if present', async () => { + const cache = mockedCache(); + cache.get.mockReturnValue(Promise.resolve('cached-hash')); + + const hash = await getFileHash(cache, sampleFilePath, stats, fd); + + expect(cache.get).toHaveBeenCalledTimes(1); + expect(generateFileHashMock).not.toHaveBeenCalled(); + expect(hash).toEqual('cached-hash'); + }); + + it('computes the value if not present in cache', async () => { + const cache = mockedCache(); + cache.get.mockReturnValue(undefined); + + generateFileHashMock.mockReturnValue(Promise.resolve('computed-hash')); + + const hash = await getFileHash(cache, sampleFilePath, stats, fd); + + expect(generateFileHashMock).toHaveBeenCalledTimes(1); + expect(generateFileHashMock).toHaveBeenCalledWith(fd); + expect(hash).toEqual('computed-hash'); + }); + + it('sets the value in the cache if not present', async () => { + const computedHashPromise = Promise.resolve('computed-hash'); + generateFileHashMock.mockReturnValue(computedHashPromise); + + const cache = mockedCache(); + cache.get.mockReturnValue(undefined); + + await getFileHash(cache, sampleFilePath, stats, fd); + + expect(cache.set).toHaveBeenCalledTimes(1); + expect(cache.set).toHaveBeenCalledWith(`${sampleFilePath}-${stats.ino}`, computedHashPromise); + }); +}); diff --git a/src/core/server/core_app/bundle_routes/file_hash.ts b/src/core/server/core_app/bundle_routes/file_hash.ts new file mode 100644 index 0000000000000..e309873254999 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/file_hash.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { Stats } from 'fs'; +import { generateFileHash, getFileCacheKey } from './utils'; +import { IFileHashCache } from './file_hash_cache'; + +/** + * Get the hash of a file via a file descriptor + */ +export async function getFileHash(cache: IFileHashCache, path: string, stat: Stats, fd: number) { + const key = getFileCacheKey(path, stat); + + const cached = cache.get(key); + if (cached) { + return await cached; + } + + const promise = generateFileHash(fd).catch((error) => { + // don't cache failed attempts + cache.del(key); + throw error; + }); + + cache.set(key, promise); + return await promise; +} diff --git a/src/core/server/core_app/bundle_routes/file_hash_cache.test.ts b/src/core/server/core_app/bundle_routes/file_hash_cache.test.ts new file mode 100644 index 0000000000000..fb519c660e637 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/file_hash_cache.test.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { FileHashCache } from './file_hash_cache'; + +describe('FileHashCache', () => { + it('returns the value stored', async () => { + const cache = new FileHashCache(); + cache.set('foo', Promise.resolve('bar')); + expect(await cache.get('foo')).toEqual('bar'); + }); + + it('can manually delete values', () => { + const cache = new FileHashCache(); + cache.set('foo', Promise.resolve('bar')); + cache.del('foo'); + expect(cache.get('foo')).toBeUndefined(); + }); + + it('only preserves a given amount of entries', async () => { + const cache = new FileHashCache(1); + cache.set('foo', Promise.resolve('bar')); + cache.set('hello', Promise.resolve('dolly')); + + expect(await cache.get('hello')).toEqual('dolly'); + expect(cache.get('foo')).toBeUndefined(); + }); +}); diff --git a/src/optimize/bundles_route/file_hash_cache.ts b/src/core/server/core_app/bundle_routes/file_hash_cache.ts similarity index 59% rename from src/optimize/bundles_route/file_hash_cache.ts rename to src/core/server/core_app/bundle_routes/file_hash_cache.ts index 9d288ccb77194..8242a5b595d60 100644 --- a/src/optimize/bundles_route/file_hash_cache.ts +++ b/src/core/server/core_app/bundle_routes/file_hash_cache.ts @@ -8,8 +8,22 @@ import LruCache from 'lru-cache'; -export class FileHashCache { - private lru = new LruCache>(100); +/** @internal */ +export interface IFileHashCache { + get(key: string): Promise | undefined; + + set(key: string, value: Promise): void; + + del(key: string): void; +} + +/** @internal */ +export class FileHashCache implements IFileHashCache { + private lru: LruCache>; + + constructor(maxSize: number = 250) { + this.lru = new LruCache(maxSize); + } get(key: string) { return this.lru.get(key); diff --git a/src/core/server/core_app/bundle_routes/fs.ts b/src/core/server/core_app/bundle_routes/fs.ts new file mode 100644 index 0000000000000..913b5c8423553 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/fs.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +// can't use fs/promises when working with streams using file descriptors +// see https://github.com/nodejs/node/issues/35862 + +import Fs from 'fs'; +import { promisify } from 'util'; + +export const open = promisify(Fs.open); +export const close = promisify(Fs.close); +export const fstat = promisify(Fs.fstat); diff --git a/src/optimize/jest.config.js b/src/core/server/core_app/bundle_routes/index.ts similarity index 77% rename from src/optimize/jest.config.js rename to src/core/server/core_app/bundle_routes/index.ts index 8469778d775a2..5b2374a74356a 100644 --- a/src/optimize/jest.config.js +++ b/src/core/server/core_app/bundle_routes/index.ts @@ -6,8 +6,4 @@ * Side Public License, v 1. */ -module.exports = { - preset: '@kbn/test', - rootDir: '../..', - roots: ['/src/optimize'], -}; +export { registerBundleRoutes } from './register_bundle_routes'; diff --git a/src/core/server/core_app/bundle_routes/register_bundle_routes.test.mocks.ts b/src/core/server/core_app/bundle_routes/register_bundle_routes.test.mocks.ts new file mode 100644 index 0000000000000..9c93f5d403c33 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/register_bundle_routes.test.mocks.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const registerRouteForBundleMock = jest.fn(); +jest.doMock('./bundles_route', () => ({ + registerRouteForBundle: registerRouteForBundleMock, +})); + +jest.doMock('@kbn/ui-shared-deps', () => ({ + distDir: 'uiSharedDepsDistDir', +})); diff --git a/src/core/server/core_app/bundle_routes/register_bundle_routes.test.ts b/src/core/server/core_app/bundle_routes/register_bundle_routes.test.ts new file mode 100644 index 0000000000000..d51c369146957 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/register_bundle_routes.test.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { registerRouteForBundleMock } from './register_bundle_routes.test.mocks'; + +import { PackageInfo } from '@kbn/config'; +import { httpServiceMock } from '../../http/http_service.mock'; +import { UiPlugins } from '../../plugins'; +import { registerBundleRoutes } from './register_bundle_routes'; +import { FileHashCache } from './file_hash_cache'; + +const createPackageInfo = (parts: Partial = {}): PackageInfo => ({ + ...parts, + buildNum: 42, + buildSha: 'sha', + dist: true, + branch: 'master', + version: '8.0.0', +}); + +const createUiPlugins = (...ids: string[]): UiPlugins => ({ + browserConfigs: new Map(), + public: new Map(), + internal: ids.reduce((map, id) => { + map.set(id, { + publicTargetDir: `/plugins/${id}/public-target-dir`, + }); + return map; + }, new Map()), +}); + +describe('registerBundleRoutes', () => { + let router: ReturnType; + + beforeEach(() => { + router = httpServiceMock.createRouter(); + }); + + afterEach(() => { + registerRouteForBundleMock.mockReset(); + }); + + it('registers core and shared-dep bundles', () => { + registerBundleRoutes({ + router, + serverBasePath: '/server-base-path', + packageInfo: createPackageInfo(), + uiPlugins: createUiPlugins(), + }); + + expect(registerRouteForBundleMock).toHaveBeenCalledTimes(2); + + expect(registerRouteForBundleMock).toHaveBeenCalledWith(router, { + fileHashCache: expect.any(FileHashCache), + isDist: true, + bundlesPath: 'uiSharedDepsDistDir', + publicPath: '/server-base-path/42/bundles/kbn-ui-shared-deps/', + routePath: '/42/bundles/kbn-ui-shared-deps/', + }); + + expect(registerRouteForBundleMock).toHaveBeenCalledWith(router, { + fileHashCache: expect.any(FileHashCache), + isDist: true, + bundlesPath: expect.stringMatching(/src\/core\/target\/public/), + publicPath: '/server-base-path/42/bundles/core/', + routePath: '/42/bundles/core/', + }); + }); + + it('registers plugin bundles', () => { + registerBundleRoutes({ + router, + serverBasePath: '/server-base-path', + packageInfo: createPackageInfo(), + uiPlugins: createUiPlugins('plugin-a', 'plugin-b'), + }); + + expect(registerRouteForBundleMock).toHaveBeenCalledTimes(4); + + expect(registerRouteForBundleMock).toHaveBeenCalledWith(router, { + fileHashCache: expect.any(FileHashCache), + isDist: true, + bundlesPath: '/plugins/plugin-a/public-target-dir', + publicPath: '/server-base-path/42/bundles/plugin/plugin-a/', + routePath: '/42/bundles/plugin/plugin-a/', + }); + + expect(registerRouteForBundleMock).toHaveBeenCalledWith(router, { + fileHashCache: expect.any(FileHashCache), + isDist: true, + bundlesPath: '/plugins/plugin-b/public-target-dir', + publicPath: '/server-base-path/42/bundles/plugin/plugin-b/', + routePath: '/42/bundles/plugin/plugin-b/', + }); + }); +}); diff --git a/src/core/server/core_app/bundle_routes/register_bundle_routes.ts b/src/core/server/core_app/bundle_routes/register_bundle_routes.ts new file mode 100644 index 0000000000000..ee54f8ef34622 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/register_bundle_routes.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { join } from 'path'; +import { PackageInfo } from '@kbn/config'; +import { distDir as uiSharedDepsDistDir } from '@kbn/ui-shared-deps'; +import { IRouter } from '../../http'; +import { UiPlugins } from '../../plugins'; +import { fromRoot } from '../../utils'; +import { FileHashCache } from './file_hash_cache'; +import { registerRouteForBundle } from './bundles_route'; + +/** + * Creates the routes that serves files from `bundlesPath`. + * + * @param {Object} options + * @property {Array<{id,path}>} options.npUiPluginPublicDirs array of ids and paths that should be served for new platform plugins + * @property {string} options.regularBundlesPath + * @property {string} options.basePublicPath + * + * @return Array.of({Hapi.Route}) + */ +export function registerBundleRoutes({ + router, + serverBasePath, // serverBasePath + uiPlugins, + packageInfo, +}: { + router: IRouter; + serverBasePath: string; + uiPlugins: UiPlugins; + packageInfo: PackageInfo; +}) { + const { dist: isDist, buildNum } = packageInfo; + // rather than calculate the fileHash on every request, we + // provide a cache object to `resolveDynamicAssetResponse()` that + // will store the most recently used hashes. + const fileHashCache = new FileHashCache(); + + registerRouteForBundle(router, { + publicPath: `${serverBasePath}/${buildNum}/bundles/kbn-ui-shared-deps/`, + routePath: `/${buildNum}/bundles/kbn-ui-shared-deps/`, + bundlesPath: uiSharedDepsDistDir, + fileHashCache, + isDist, + }); + registerRouteForBundle(router, { + publicPath: `${serverBasePath}/${buildNum}/bundles/core/`, + routePath: `/${buildNum}/bundles/core/`, + bundlesPath: fromRoot(join('src', 'core', 'target', 'public')), + fileHashCache, + isDist, + }); + + [...uiPlugins.internal.entries()].forEach(([id, { publicTargetDir }]) => { + registerRouteForBundle(router, { + publicPath: `${serverBasePath}/${buildNum}/bundles/plugin/${id}/`, + routePath: `/${buildNum}/bundles/plugin/${id}/`, + bundlesPath: publicTargetDir, + fileHashCache, + isDist, + }); + }); +} diff --git a/src/core/server/core_app/bundle_routes/select_compressed_file.ts b/src/core/server/core_app/bundle_routes/select_compressed_file.ts new file mode 100644 index 0000000000000..c7b071a9c3548 --- /dev/null +++ b/src/core/server/core_app/bundle_routes/select_compressed_file.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { extname } from 'path'; +import Accept from 'accept'; +import { open } from './fs'; + +declare module 'accept' { + // @types/accept does not include the `preferences` argument so we override the type to include it + export function encodings(encodingHeader?: string, preferences?: string[]): string[]; +} + +async function tryToOpenFile(filePath: string) { + try { + return await open(filePath, 'r'); + } catch (e) { + if (e.code === 'ENOENT') { + return undefined; + } else { + throw e; + } + } +} + +export async function selectCompressedFile(acceptEncodingHeader: string | undefined, path: string) { + let fd: number | undefined; + let fileEncoding: 'gzip' | 'br' | undefined; + const ext = extname(path); + + const supportedEncodings = Accept.encodings(acceptEncodingHeader, ['br', 'gzip']); + + // do not bother trying to look compressed versions for anything else than js or css files + if (ext === '.js' || ext === '.css') { + if (supportedEncodings[0] === 'br') { + fileEncoding = 'br'; + fd = await tryToOpenFile(`${path}.br`); + } + if (!fd && supportedEncodings.includes('gzip')) { + fileEncoding = 'gzip'; + fd = await tryToOpenFile(`${path}.gz`); + } + } + + if (!fd) { + fileEncoding = undefined; + // Use raw open to trigger exception if it does not exist + fd = await open(path, 'r'); + } + + return { fd, fileEncoding }; +} diff --git a/src/optimize/bundles_route/file_hash.ts b/src/core/server/core_app/bundle_routes/utils.ts similarity index 51% rename from src/optimize/bundles_route/file_hash.ts rename to src/core/server/core_app/bundle_routes/utils.ts index 1f5b1a979407c..a2adefcfa73c2 100644 --- a/src/optimize/bundles_route/file_hash.ts +++ b/src/core/server/core_app/bundle_routes/utils.ts @@ -6,33 +6,19 @@ * Side Public License, v 1. */ +import { createReadStream, Stats } from 'fs'; import { createHash } from 'crypto'; -import Fs from 'fs'; - import * as Rx from 'rxjs'; -import { takeUntil, map } from 'rxjs/operators'; - -import { FileHashCache } from './file_hash_cache'; - -/** - * Get the hash of a file via a file descriptor - */ -export async function getFileHash(cache: FileHashCache, path: string, stat: Fs.Stats, fd: number) { - const key = `${path}:${stat.ino}:${stat.size}:${stat.mtime.getTime()}`; - - const cached = cache.get(key); - if (cached) { - return await cached; - } +import { map, takeUntil } from 'rxjs/operators'; +export const generateFileHash = (fd: number): Promise => { const hash = createHash('sha1'); - const read = Fs.createReadStream(null as any, { + const read = createReadStream(null as any, { fd, start: 0, autoClose: false, }); - - const promise = Rx.merge( + return Rx.merge( Rx.fromEvent(read, 'data'), Rx.fromEvent(read, 'error').pipe( map((error) => { @@ -42,13 +28,8 @@ export async function getFileHash(cache: FileHashCache, path: string, stat: Fs.S ) .pipe(takeUntil(Rx.fromEvent(read, 'end'))) .forEach((chunk) => hash.update(chunk)) - .then(() => hash.digest('hex')) - .catch((error) => { - // don't cache failed attempts - cache.del(key); - throw error; - }); + .then(() => hash.digest('hex')); +}; - cache.set(key, promise); - return await promise; -} +export const getFileCacheKey = (path: string, stat: Stats) => + `${path}:${stat.ino}:${stat.size}:${stat.mtime.getTime()}`; diff --git a/src/optimize/bundles_route/index.ts b/src/core/server/core_app/core_app.test.mocks.ts similarity index 70% rename from src/optimize/bundles_route/index.ts rename to src/core/server/core_app/core_app.test.mocks.ts index 086bce552c5d0..d45df8dd52d71 100644 --- a/src/optimize/bundles_route/index.ts +++ b/src/core/server/core_app/core_app.test.mocks.ts @@ -6,5 +6,7 @@ * Side Public License, v 1. */ -export { createBundlesRoute } from './bundles_route'; -export { createProxyBundlesRoute } from './proxy_bundles_route'; +export const registerBundleRoutesMock = jest.fn(); +jest.doMock('./bundle_routes', () => ({ + registerBundleRoutes: registerBundleRoutesMock, +})); diff --git a/src/core/server/core_app/core_app.test.ts b/src/core/server/core_app/core_app.test.ts index e08a8e0be0a41..ad7af3ac8b84d 100644 --- a/src/core/server/core_app/core_app.test.ts +++ b/src/core/server/core_app/core_app.test.ts @@ -6,28 +6,42 @@ * Side Public License, v 1. */ +import { registerBundleRoutesMock } from './core_app.test.mocks'; + import { mockCoreContext } from '../core_context.mock'; import { coreMock } from '../mocks'; import { httpResourcesMock } from '../http_resources/http_resources_service.mock'; +import type { UiPlugins } from '../plugins'; import { CoreApp } from './core_app'; +const emptyPlugins = (): UiPlugins => ({ + internal: new Map(), + public: new Map(), + browserConfigs: new Map(), +}); + describe('CoreApp', () => { + let coreContext: ReturnType; let coreApp: CoreApp; let internalCoreSetup: ReturnType; let httpResourcesRegistrar: ReturnType; beforeEach(() => { - const coreContext = mockCoreContext.create(); + coreContext = mockCoreContext.create(); internalCoreSetup = coreMock.createInternalSetup(); httpResourcesRegistrar = httpResourcesMock.createRegistrar(); internalCoreSetup.httpResources.createRegistrar.mockReturnValue(httpResourcesRegistrar); coreApp = new CoreApp(coreContext); }); + afterEach(() => { + registerBundleRoutesMock.mockReset(); + }); + describe('`/status` route', () => { it('is registered with `authRequired: false` is the status page is anonymous', () => { internalCoreSetup.status.isStatusPageAnonymous.mockReturnValue(true); - coreApp.setup(internalCoreSetup); + coreApp.setup(internalCoreSetup, emptyPlugins()); expect(httpResourcesRegistrar.register).toHaveBeenCalledWith( { @@ -43,7 +57,7 @@ describe('CoreApp', () => { it('is registered with `authRequired: true` is the status page is not anonymous', () => { internalCoreSetup.status.isStatusPageAnonymous.mockReturnValue(false); - coreApp.setup(internalCoreSetup); + coreApp.setup(internalCoreSetup, emptyPlugins()); expect(httpResourcesRegistrar.register).toHaveBeenCalledWith( { @@ -60,7 +74,7 @@ describe('CoreApp', () => { describe('`/app/{id}/{any*}` route', () => { it('is registered with the correct parameters', () => { - coreApp.setup(internalCoreSetup); + coreApp.setup(internalCoreSetup, emptyPlugins()); expect(httpResourcesRegistrar.register).toHaveBeenCalledWith( { @@ -74,4 +88,17 @@ describe('CoreApp', () => { ); }); }); + + it('calls `registerBundleRoutes` with the correct options', () => { + const uiPlugins = emptyPlugins(); + coreApp.setup(internalCoreSetup, uiPlugins); + + expect(registerBundleRoutesMock).toHaveBeenCalledTimes(1); + expect(registerBundleRoutesMock).toHaveBeenCalledWith({ + uiPlugins, + router: expect.any(Object), + packageInfo: coreContext.env.packageInfo, + serverBasePath: internalCoreSetup.http.basePath.serverBasePath, + }); + }); }); diff --git a/src/core/server/core_app/core_app.ts b/src/core/server/core_app/core_app.ts index 24ddc305d8232..dac941767ebb5 100644 --- a/src/core/server/core_app/core_app.ts +++ b/src/core/server/core_app/core_app.ts @@ -7,27 +7,32 @@ */ import Path from 'path'; -import { fromRoot } from '../../../core/server/utils'; +import { Env } from '@kbn/config'; +import { fromRoot } from '../utils'; import { InternalCoreSetup } from '../internal_types'; import { CoreContext } from '../core_context'; import { Logger } from '../logging'; +import { registerBundleRoutes } from './bundle_routes'; +import { UiPlugins } from '../plugins'; /** @internal */ export class CoreApp { private readonly logger: Logger; + private readonly env: Env; constructor(core: CoreContext) { this.logger = core.logger.get('core-app'); + this.env = core.env; } - setup(coreSetup: InternalCoreSetup) { + setup(coreSetup: InternalCoreSetup, uiPlugins: UiPlugins) { this.logger.debug('Setting up core app.'); - this.registerDefaultRoutes(coreSetup); + this.registerDefaultRoutes(coreSetup, uiPlugins); this.registerStaticDirs(coreSetup); } - private registerDefaultRoutes(coreSetup: InternalCoreSetup) { + private registerDefaultRoutes(coreSetup: InternalCoreSetup, uiPlugins: UiPlugins) { const httpSetup = coreSetup.http; const router = httpSetup.createRouter(''); const resources = coreSetup.httpResources.createRegistrar(router); @@ -48,6 +53,13 @@ export class CoreApp { res.ok({ body: { version: '0.0.1' } }) ); + registerBundleRoutes({ + router, + uiPlugins, + packageInfo: this.env.packageInfo, + serverBasePath: coreSetup.http.basePath.serverBasePath, + }); + resources.register( { path: '/app/{id}/{any*}', diff --git a/src/optimize/bundles_route/__fixtures__/outside_output.js b/src/core/server/core_app/integration_tests/__fixtures__/outside_output.js similarity index 100% rename from src/optimize/bundles_route/__fixtures__/outside_output.js rename to src/core/server/core_app/integration_tests/__fixtures__/outside_output.js diff --git a/src/optimize/index.ts b/src/core/server/core_app/integration_tests/__fixtures__/plugin/foo/gzip_chunk.js similarity index 87% rename from src/optimize/index.ts rename to src/core/server/core_app/integration_tests/__fixtures__/plugin/foo/gzip_chunk.js index 3073c62d55b40..ca84988e8f978 100644 --- a/src/optimize/index.ts +++ b/src/core/server/core_app/integration_tests/__fixtures__/plugin/foo/gzip_chunk.js @@ -6,4 +6,4 @@ * Side Public License, v 1. */ -export { optimizeMixin } from './optimize_mixin'; +module.exports = 'GZIP-CHUNK'; diff --git a/src/core/server/core_app/integration_tests/__fixtures__/plugin/foo/gzip_chunk.js.gz b/src/core/server/core_app/integration_tests/__fixtures__/plugin/foo/gzip_chunk.js.gz new file mode 100644 index 0000000000000000000000000000000000000000..fbf388e74ee70d5575ba071c2a6f091141bcea7c GIT binary patch literal 256 zcmV+b0ssCViwFqR#YtcQ17~_^a4lnKb#7}eYI6W}j=@gDFc3uV`HHz!s)7StD=t(B z5C~PO;K0pwmS!c#8`YS>u446-LSE?T8?XJl$!>31939baXUm0dknk`oXgej_}Re)v#Ko z5#-4!OQU&23 literal 0 HcmV?d00001 diff --git a/src/optimize/bundles_route/__fixtures__/plugin/foo/image.png b/src/core/server/core_app/integration_tests/__fixtures__/plugin/foo/image.png similarity index 100% rename from src/optimize/bundles_route/__fixtures__/plugin/foo/image.png rename to src/core/server/core_app/integration_tests/__fixtures__/plugin/foo/image.png diff --git a/src/optimize/bundles_route/__fixtures__/plugin/foo/plugin.js b/src/core/server/core_app/integration_tests/__fixtures__/plugin/foo/plugin.js similarity index 100% rename from src/optimize/bundles_route/__fixtures__/plugin/foo/plugin.js rename to src/core/server/core_app/integration_tests/__fixtures__/plugin/foo/plugin.js diff --git a/src/core/server/core_app/integration_tests/bundle_routes.test.ts b/src/core/server/core_app/integration_tests/bundle_routes.test.ts new file mode 100644 index 0000000000000..fbe2e9285ba29 --- /dev/null +++ b/src/core/server/core_app/integration_tests/bundle_routes.test.ts @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { resolve } from 'path'; +import { readFile } from 'fs/promises'; +import supertest from 'supertest'; +import { contextServiceMock } from '../../context/context_service.mock'; +import { loggingSystemMock } from '../../logging/logging_system.mock'; +import { HttpService, IRouter } from '../../http'; +import { createHttpServer } from '../../http/test_utils'; +import { registerRouteForBundle } from '../bundle_routes/bundles_route'; +import { FileHashCache } from '../bundle_routes/file_hash_cache'; + +const buildNum = 1234; +const fooPluginFixture = resolve(__dirname, './__fixtures__/plugin/foo'); + +describe('bundle routes', () => { + let server: HttpService; + let contextSetup: ReturnType; + let logger: ReturnType; + let fileHashCache: FileHashCache; + + beforeEach(() => { + contextSetup = contextServiceMock.createSetupContract(); + logger = loggingSystemMock.create(); + fileHashCache = new FileHashCache(); + + server = createHttpServer({ logger }); + }); + + afterEach(async () => { + await server.stop(); + }); + + const registerFooPluginRoute = ( + router: IRouter, + { isDist = false }: { isDist?: boolean } = {} + ) => { + registerRouteForBundle(router, { + isDist, + fileHashCache, + bundlesPath: fooPluginFixture, + routePath: `/${buildNum}/bundles/plugin/foo/`, + publicPath: `/${buildNum}/bundles/plugin/foo/`, + }); + }; + + it('serves images inside from the bundle path', async () => { + const { server: innerServer, createRouter } = await server.setup({ + context: contextSetup, + }); + + registerFooPluginRoute(createRouter('')); + await server.start(); + + const response = await supertest(innerServer.listener) + .get(`/${buildNum}/bundles/plugin/foo/image.png`) + .expect(200); + + const actualImage = await readFile(resolve(fooPluginFixture, 'image.png')); + expect(response.get('content-type')).toEqual('image/png'); + expect(response.body).toEqual(actualImage); + }); + + it('serves uncompressed js files', async () => { + const { server: innerServer, createRouter } = await server.setup({ + context: contextSetup, + }); + + registerFooPluginRoute(createRouter('')); + await server.start(); + + const response = await supertest(innerServer.listener) + .get(`/${buildNum}/bundles/plugin/foo/plugin.js`) + .expect(200); + + const actualFile = await readFile(resolve(fooPluginFixture, 'plugin.js')); + expect(response.get('content-type')).toEqual('application/javascript; charset=utf-8'); + expect(actualFile.toString('utf8')).toEqual(response.text); + }); + + it('returns 404 for files outside of the bundlePath', async () => { + const { server: innerServer, createRouter } = await server.setup({ + context: contextSetup, + }); + + registerFooPluginRoute(createRouter('')); + await server.start(); + + await supertest(innerServer.listener) + .get(`/${buildNum}/bundles/plugin/foo/../outside_output.js`) + .expect(404); + }); + + it('returns 404 for non-existing files', async () => { + const { server: innerServer, createRouter } = await server.setup({ + context: contextSetup, + }); + + registerFooPluginRoute(createRouter('')); + await server.start(); + + await supertest(innerServer.listener) + .get(`/${buildNum}/bundles/plugin/foo/missing.js`) + .expect(404); + }); + + it('returns gzip version if present', async () => { + const { server: innerServer, createRouter } = await server.setup({ + context: contextSetup, + }); + + registerFooPluginRoute(createRouter('')); + await server.start(); + + const response = await supertest(innerServer.listener) + .get(`/${buildNum}/bundles/plugin/foo/gzip_chunk.js`) + .expect(200); + + expect(response.get('content-encoding')).toEqual('gzip'); + expect(response.get('content-type')).toEqual('application/javascript; charset=utf-8'); + + const actualFile = await readFile(resolve(fooPluginFixture, 'gzip_chunk.js')); + expect(actualFile.toString('utf8')).toEqual(response.text); + }); + + // supertest does not support brotli compression, cannot test + // this is covered in FTR tests anyway + it.skip('returns br version if present', () => {}); + + describe('in production mode', () => { + it('uses max-age cache-control', async () => { + const { server: innerServer, createRouter } = await server.setup({ + context: contextSetup, + }); + + registerFooPluginRoute(createRouter(''), { isDist: true }); + await server.start(); + + const response = await supertest(innerServer.listener) + .get(`/${buildNum}/bundles/plugin/foo/gzip_chunk.js`) + .expect(200); + + expect(response.get('cache-control')).toEqual('max-age=31536000'); + expect(response.get('etag')).toBeUndefined(); + }); + }); + + describe('in development mode', () => { + it('uses etag cache-control', async () => { + const { server: innerServer, createRouter } = await server.setup({ + context: contextSetup, + }); + + registerFooPluginRoute(createRouter(''), { isDist: false }); + await server.start(); + + const response = await supertest(innerServer.listener) + .get(`/${buildNum}/bundles/plugin/foo/gzip_chunk.js`) + .expect(200); + + expect(response.get('cache-control')).toEqual('must-revalidate'); + expect(response.get('etag')).toBeDefined(); + }); + }); +}); diff --git a/src/core/server/server.ts b/src/core/server/server.ts index 337dfa8824303..ef5164a8c48e1 100644 --- a/src/core/server/server.ts +++ b/src/core/server/server.ts @@ -221,7 +221,7 @@ export class Server { }); this.registerCoreContext(coreSetup); - this.coreApp.setup(coreSetup); + this.coreApp.setup(coreSetup, uiPlugins); setupTransaction?.end(); return coreSetup; diff --git a/src/legacy/server/kbn_server.js b/src/legacy/server/kbn_server.js index 55593d13d4687..d2eebb7b0cd23 100644 --- a/src/legacy/server/kbn_server.js +++ b/src/legacy/server/kbn_server.js @@ -15,7 +15,6 @@ import { Config } from './config'; import httpMixin from './http'; import { coreMixin } from './core'; import { loggingMixin } from './logging'; -import { optimizeMixin } from '../../optimize'; /** * @typedef {import('./kbn_server').KibanaConfig} KibanaConfig @@ -63,10 +62,7 @@ export default class KbnServer { coreMixin, - loggingMixin, - - // setup routes that serve the @kbn/optimizer output - optimizeMixin + loggingMixin ) ); diff --git a/src/optimize/bundles_route/bundles_route.test.ts b/src/optimize/bundles_route/bundles_route.test.ts deleted file mode 100644 index 4a5af40a66cfb..0000000000000 --- a/src/optimize/bundles_route/bundles_route.test.ts +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { resolve } from 'path'; -import { readFileSync } from 'fs'; -import crypto from 'crypto'; - -import Chance from 'chance'; -import Hapi from '@hapi/hapi'; -import Inert from '@hapi/inert'; - -import { createBundlesRoute } from './bundles_route'; - -const chance = new Chance(); -const fooPluginFixture = resolve(__dirname, './__fixtures__/plugin/foo'); -const createHashMock = jest.spyOn(crypto, 'createHash'); - -const randomWordsCache = new Set(); -const uniqueRandomWord = (): string => { - const word = chance.word(); - - if (randomWordsCache.has(word)) { - return uniqueRandomWord(); - } - - randomWordsCache.add(word); - return word; -}; - -function createServer({ - basePublicPath = '', - isDist = false, -}: { - basePublicPath?: string; - isDist?: boolean; -} = {}) { - const buildHash = '1234'; - const npUiPluginPublicDirs = [ - { - id: 'foo', - path: fooPluginFixture, - }, - ]; - - const server = new Hapi.Server(); - server.register([Inert]); - - server.route( - createBundlesRoute({ - basePublicPath, - npUiPluginPublicDirs, - buildHash, - isDist, - }) - ); - - return server; -} - -beforeEach(() => { - jest.clearAllMocks(); -}); - -describe('validation', () => { - it('validates that basePublicPath is valid', () => { - expect(() => { - createServer({ - // @ts-expect-error intentionally trying to break things - basePublicPath: 123, - }); - }).toThrowErrorMatchingInlineSnapshot(`"basePublicPath must be a string"`); - expect(() => { - createServer({ - // @ts-expect-error intentionally trying to break things - basePublicPath: {}, - }); - }).toThrowErrorMatchingInlineSnapshot(`"basePublicPath must be a string"`); - expect(() => { - createServer({ - basePublicPath: '/a/', - }); - }).toThrowErrorMatchingInlineSnapshot( - `"basePublicPath must be empty OR start and not end with a /"` - ); - expect(() => { - createServer({ - basePublicPath: 'a/', - }); - }).toThrowErrorMatchingInlineSnapshot( - `"basePublicPath must be empty OR start and not end with a /"` - ); - expect(() => { - createServer({ - basePublicPath: '/a', - }); - }).not.toThrowError(); - expect(() => { - createServer({ - basePublicPath: '', - }); - }).not.toThrowError(); - }); -}); - -describe('image', () => { - it('responds with exact file data', async () => { - const server = createServer(); - const response = await server.inject({ - url: '/1234/bundles/plugin/foo/image.png', - }); - - expect(response.statusCode).toBe(200); - const image = readFileSync(resolve(fooPluginFixture, 'image.png')); - expect(response.headers).toHaveProperty('content-length', image.length); - expect(response.headers).toHaveProperty('content-type', 'image/png'); - expect(image).toEqual(response.rawPayload); - }); -}); - -describe('js file', () => { - it('responds with no content-length and exact file data', async () => { - const server = createServer(); - const response = await server.inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - }); - - expect(response.statusCode).toBe(200); - expect(response.headers).not.toHaveProperty('content-length'); - expect(response.headers).toHaveProperty( - 'content-type', - 'application/javascript; charset=utf-8' - ); - expect(readFileSync(resolve(fooPluginFixture, 'plugin.js'))).toEqual(response.rawPayload); - }); -}); - -describe('js file outside plugin', () => { - it('responds with a 404', async () => { - const server = createServer(); - - const response = await server.inject({ - url: '/1234/bundles/plugin/foo/../outside_output.js', - }); - - expect(response.statusCode).toBe(404); - expect(response.result).toEqual({ - error: 'Not Found', - message: 'Not Found', - statusCode: 404, - }); - }); -}); - -describe('missing js file', () => { - it('responds with 404', async () => { - const server = createServer(); - - const response = await server.inject({ - url: '/1234/bundles/plugin/foo/non_existent.js', - }); - - expect(response.statusCode).toBe(404); - expect(response.result).toEqual({ - error: 'Not Found', - message: 'Not Found', - statusCode: 404, - }); - }); -}); - -describe('etag', () => { - it('only calculates hash of file on first request', async () => { - const server = createServer(); - - expect(createHashMock).not.toHaveBeenCalled(); - const resp1 = await server.inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - }); - - expect(createHashMock).toHaveBeenCalledTimes(1); - createHashMock.mockClear(); - expect(resp1.statusCode).toBe(200); - - const resp2 = await server.inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - }); - - expect(createHashMock).not.toHaveBeenCalled(); - expect(resp2.statusCode).toBe(200); - }); - - it('is unique per basePublicPath although content is the same (by default)', async () => { - const basePublicPath1 = `/${uniqueRandomWord()}`; - const basePublicPath2 = `/${uniqueRandomWord()}`; - - const [resp1, resp2] = await Promise.all([ - createServer({ basePublicPath: basePublicPath1 }).inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - }), - createServer({ basePublicPath: basePublicPath2 }).inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - }), - ]); - - expect(resp1.statusCode).toBe(200); - expect(resp2.statusCode).toBe(200); - - expect(resp1.rawPayload).toEqual(resp2.rawPayload); - - expect(resp1.headers.etag).toEqual(expect.any(String)); - expect(resp2.headers.etag).toEqual(expect.any(String)); - expect(resp1.headers.etag).not.toEqual(resp2.headers.etag); - }); -}); - -describe('cache control', () => { - it('responds with 304 when etag and last modified are sent back', async () => { - const server = createServer(); - const resp = await server.inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - }); - - expect(resp.statusCode).toBe(200); - - const resp2 = await server.inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - headers: { - 'if-modified-since': resp.headers['last-modified'], - 'if-none-match': resp.headers.etag, - }, - }); - - expect(resp2.statusCode).toBe(304); - expect(resp2.result).toHaveLength(0); - }); -}); - -describe('caching', () => { - describe('for non-distributable mode', () => { - it('uses "etag" header to invalidate cache', async () => { - const basePublicPath = `/${uniqueRandomWord()}`; - - const responce = await createServer({ basePublicPath }).inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - }); - - expect(responce.statusCode).toBe(200); - - expect(responce.headers.etag).toEqual(expect.any(String)); - expect(responce.headers['cache-control']).toBe('must-revalidate'); - }); - - it('creates the same "etag" header for the same content with the same basePath', async () => { - const [resp1, resp2] = await Promise.all([ - createServer({ basePublicPath: '' }).inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - }), - createServer({ basePublicPath: '' }).inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - }), - ]); - - expect(resp1.statusCode).toBe(200); - expect(resp2.statusCode).toBe(200); - - expect(resp1.rawPayload).toEqual(resp2.rawPayload); - - expect(resp1.headers.etag).toEqual(expect.any(String)); - expect(resp2.headers.etag).toEqual(expect.any(String)); - expect(resp1.headers.etag).toEqual(resp2.headers.etag); - }); - }); - - describe('for distributable mode', () => { - it('commands to cache assets for each release for a year', async () => { - const basePublicPath = `/${uniqueRandomWord()}`; - - const responce = await createServer({ - basePublicPath, - isDist: true, - }).inject({ - url: '/1234/bundles/plugin/foo/plugin.js', - }); - - expect(responce.statusCode).toBe(200); - - expect(responce.headers.etag).toBe(undefined); - expect(responce.headers['cache-control']).toBe('max-age=31536000'); - }); - }); -}); diff --git a/src/optimize/bundles_route/bundles_route.ts b/src/optimize/bundles_route/bundles_route.ts deleted file mode 100644 index b88ca7e5c22b1..0000000000000 --- a/src/optimize/bundles_route/bundles_route.ts +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { extname, join } from 'path'; - -import Hapi from '@hapi/hapi'; -import * as UiSharedDeps from '@kbn/ui-shared-deps'; -import agent from 'elastic-apm-node'; - -import { createDynamicAssetResponse } from './dynamic_asset_response'; -import { FileHashCache } from './file_hash_cache'; -import { assertIsNpUiPluginPublicDirs, NpUiPluginPublicDirs } from '../np_ui_plugin_public_dirs'; -import { fromRoot } from '../../core/server/utils'; - -/** - * Creates the routes that serves files from `bundlesPath`. - * - * @param {Object} options - * @property {Array<{id,path}>} options.npUiPluginPublicDirs array of ids and paths that should be served for new platform plugins - * @property {string} options.regularBundlesPath - * @property {string} options.basePublicPath - * - * @return Array.of({Hapi.Route}) - */ -export function createBundlesRoute({ - basePublicPath, - npUiPluginPublicDirs = [], - buildHash, - isDist = false, -}: { - basePublicPath: string; - npUiPluginPublicDirs?: NpUiPluginPublicDirs; - buildHash: string; - isDist?: boolean; -}) { - // rather than calculate the fileHash on every request, we - // provide a cache object to `resolveDynamicAssetResponse()` that - // will store the 100 most recently used hashes. - const fileHashCache = new FileHashCache(); - assertIsNpUiPluginPublicDirs(npUiPluginPublicDirs); - - if (typeof basePublicPath !== 'string') { - throw new TypeError('basePublicPath must be a string'); - } - - if (!basePublicPath.match(/(^$|^\/.*[^\/]$)/)) { - throw new TypeError('basePublicPath must be empty OR start and not end with a /'); - } - - return [ - buildRouteForBundles({ - publicPath: `${basePublicPath}/${buildHash}/bundles/kbn-ui-shared-deps/`, - routePath: `/${buildHash}/bundles/kbn-ui-shared-deps/`, - bundlesPath: UiSharedDeps.distDir, - fileHashCache, - isDist, - }), - ...npUiPluginPublicDirs.map(({ id, path }) => - buildRouteForBundles({ - publicPath: `${basePublicPath}/${buildHash}/bundles/plugin/${id}/`, - routePath: `/${buildHash}/bundles/plugin/${id}/`, - bundlesPath: path, - fileHashCache, - isDist, - }) - ), - buildRouteForBundles({ - publicPath: `${basePublicPath}/${buildHash}/bundles/core/`, - routePath: `/${buildHash}/bundles/core/`, - bundlesPath: fromRoot(join('src', 'core', 'target', 'public')), - fileHashCache, - isDist, - }), - ]; -} - -function buildRouteForBundles({ - publicPath, - routePath, - bundlesPath, - fileHashCache, - isDist, -}: { - publicPath: string; - routePath: string; - bundlesPath: string; - fileHashCache: FileHashCache; - isDist: boolean; -}) { - return { - method: 'GET', - path: `${routePath}{path*}`, - config: { - auth: false, - ext: { - onPreHandler: { - method(request: Hapi.Request, h: Hapi.ResponseToolkit) { - const ext = extname(request.params.path); - - agent.setTransactionName('GET ?/bundles/?'); - - if (ext !== '.js' && ext !== '.css') { - return h.continue; - } - - return createDynamicAssetResponse({ - request, - h, - bundlesPath, - fileHashCache, - publicPath, - isDist, - }); - }, - }, - }, - }, - handler: { - directory: { - path: bundlesPath, - listing: false, - lookupCompressed: true, - }, - }, - }; -} diff --git a/src/optimize/bundles_route/dynamic_asset_response.ts b/src/optimize/bundles_route/dynamic_asset_response.ts deleted file mode 100644 index 309fe6dd47d51..0000000000000 --- a/src/optimize/bundles_route/dynamic_asset_response.ts +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import Fs from 'fs'; -import { resolve } from 'path'; -import { promisify } from 'util'; - -import Accept from 'accept'; -import Boom from '@hapi/boom'; -import Hapi from '@hapi/hapi'; - -import { FileHashCache } from './file_hash_cache'; -import { getFileHash } from './file_hash'; - -const MINUTE = 60; -const HOUR = 60 * MINUTE; -const DAY = 24 * HOUR; - -const asyncOpen = promisify(Fs.open); -const asyncClose = promisify(Fs.close); -const asyncFstat = promisify(Fs.fstat); - -async function tryToOpenFile(filePath: string) { - try { - return await asyncOpen(filePath, 'r'); - } catch (e) { - if (e.code === 'ENOENT') { - return undefined; - } else { - throw e; - } - } -} - -async function selectCompressedFile(acceptEncodingHeader: string | undefined, path: string) { - let fd: number | undefined; - let fileEncoding: 'gzip' | 'br' | undefined; - - const supportedEncodings = Accept.encodings(acceptEncodingHeader, ['br', 'gzip']); - - if (supportedEncodings[0] === 'br') { - fileEncoding = 'br'; - fd = await tryToOpenFile(`${path}.br`); - } - if (!fd && supportedEncodings.includes('gzip')) { - fileEncoding = 'gzip'; - fd = await tryToOpenFile(`${path}.gz`); - } - if (!fd) { - fileEncoding = undefined; - // Use raw open to trigger exception if it does not exist - fd = await asyncOpen(path, 'r'); - } - - return { fd, fileEncoding }; -} - -/** - * Create a Hapi response for the requested path. This is designed - * to replicate a subset of the features provided by Hapi's Inert - * plugin including: - * - ensure path is not traversing out of the bundle directory - * - manage use file descriptors for file access to efficiently - * interact with the file multiple times in each request - * - generate and cache etag for the file - * - write correct headers to response for client-side caching - * and invalidation - * - stream file to response - * - * It differs from Inert in some important ways: - * - cached hash/etag is based on the file on disk, but modified - * by the public path so that individual public paths have - * different etags, but can share a cache - */ -export async function createDynamicAssetResponse({ - request, - h, - bundlesPath, - publicPath, - fileHashCache, - isDist, -}: { - request: Hapi.Request; - h: Hapi.ResponseToolkit; - bundlesPath: string; - publicPath: string; - fileHashCache: FileHashCache; - isDist: boolean; -}) { - let fd: number | undefined; - let fileEncoding: 'gzip' | 'br' | undefined; - - try { - const path = resolve(bundlesPath, request.params.path); - - // prevent path traversal, only process paths that resolve within bundlesPath - if (!path.startsWith(bundlesPath)) { - throw Boom.forbidden(undefined, 'EACCES'); - } - - // we use and manage a file descriptor mostly because - // that's what Inert does, and since we are accessing - // the file 2 or 3 times per request it seems logical - ({ fd, fileEncoding } = await selectCompressedFile(request.headers['accept-encoding'], path)); - - const stat = await asyncFstat(fd); - const hash = isDist ? undefined : await getFileHash(fileHashCache, path, stat, fd); - - const content = Fs.createReadStream(null as any, { - fd, - start: 0, - autoClose: true, - }); - fd = undefined; // read stream is now responsible for fd - - const response = h - .response(content) - .takeover() - .code(200) - .type(request.server.mime.path(path).type); - - if (isDist) { - response.header('cache-control', `max-age=${365 * DAY}`); - } else { - response.etag(`${hash}-${publicPath}`); - response.header('cache-control', 'must-revalidate'); - } - - // If we manually selected a compressed file, specify the encoding header. - // Otherwise, let Hapi automatically gzip the response. - if (fileEncoding) { - response.header('content-encoding', fileEncoding); - } - - return response; - } catch (error) { - if (fd) { - try { - await asyncClose(fd); - } catch (_) { - // ignore errors from close, we already have one to report - // and it's very likely they are the same - } - } - - if (error.code === 'ENOENT') { - throw Boom.notFound(); - } - - throw Boom.boomify(error); - } -} diff --git a/src/optimize/bundles_route/proxy_bundles_route.ts b/src/optimize/bundles_route/proxy_bundles_route.ts deleted file mode 100644 index cb7f326b961f5..0000000000000 --- a/src/optimize/bundles_route/proxy_bundles_route.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -export function createProxyBundlesRoute({ - host, - port, - buildHash, -}: { - host: string; - port: number; - buildHash: string; -}) { - return [buildProxyRouteForBundles(`/${buildHash}/bundles/`, host, port)]; -} - -function buildProxyRouteForBundles(routePath: string, host: string, port: number) { - return { - path: `${routePath}{path*}`, - method: 'GET', - handler: { - proxy: { - host, - port, - passThrough: true, - xforward: true, - }, - }, - config: { auth: false }, - }; -} diff --git a/src/optimize/np_ui_plugin_public_dirs.ts b/src/optimize/np_ui_plugin_public_dirs.ts deleted file mode 100644 index c5a4b8b85ce49..0000000000000 --- a/src/optimize/np_ui_plugin_public_dirs.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import KbnServer from '../legacy/server/kbn_server'; - -export type NpUiPluginPublicDirs = Array<{ - id: string; - path: string; -}>; - -export function getNpUiPluginPublicDirs(kbnServer: KbnServer): NpUiPluginPublicDirs { - return Array.from(kbnServer.newPlatform.__internals.uiPlugins.internal.entries()).map( - ([id, { publicTargetDir }]) => ({ - id, - path: publicTargetDir, - }) - ); -} - -export function isNpUiPluginPublicDirs(x: any): x is NpUiPluginPublicDirs { - return ( - Array.isArray(x) && - x.every( - (s) => typeof s === 'object' && s && typeof s.id === 'string' && typeof s.path === 'string' - ) - ); -} - -export function assertIsNpUiPluginPublicDirs(x: any): asserts x is NpUiPluginPublicDirs { - if (!isNpUiPluginPublicDirs(x)) { - throw new TypeError( - 'npUiPluginPublicDirs must be an array of objects with string `id` and `path` properties' - ); - } -} diff --git a/src/optimize/optimize_mixin.ts b/src/optimize/optimize_mixin.ts deleted file mode 100644 index dc780b0fae44c..0000000000000 --- a/src/optimize/optimize_mixin.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import Hapi from '@hapi/hapi'; - -import { createBundlesRoute } from './bundles_route'; -import { getNpUiPluginPublicDirs } from './np_ui_plugin_public_dirs'; -import KbnServer, { KibanaConfig } from '../legacy/server/kbn_server'; - -export const optimizeMixin = async ( - kbnServer: KbnServer, - server: Hapi.Server, - config: KibanaConfig -) => { - server.route( - createBundlesRoute({ - basePublicPath: config.get('server.basePath'), - npUiPluginPublicDirs: getNpUiPluginPublicDirs(kbnServer), - buildHash: kbnServer.newPlatform.env.packageInfo.buildNum.toString(), - isDist: kbnServer.newPlatform.env.packageInfo.dist, - }) - ); -};