diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml index 81f04f933208c..aae27bd38af0f 100644 --- a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml +++ b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml @@ -5,7 +5,7 @@ # SKIP_VERIFICATION: if set to 1/true, it will skip running all tests # SKIP_CYPRESS: if set to 1/true, it will skip running the cypress tests # FTR_EXTRA_ARGS: a string argument, if passed, it will be forwarded verbatim to the FTR run script -# ES_SERVERLESS_IMAGE: the tag for the docker image to test, in the form of docker.elastic.co/elasticsearch-ci/elasticsearch-serverless:$TAG +# ES_SERVERLESS_IMAGE: the full image path for the docker image to test # BUILDKITE_COMMIT: the commit hash of the kibana branch to test agents: @@ -16,16 +16,7 @@ agents: steps: - label: "Annotate runtime parameters" - command: | - buildkite-agent annotate --context kibana-commit --style info "Kibana build hash: $BUILDKITE_BRANCH / $BUILDKITE_COMMIT" - cat << EOF | buildkite-agent annotate --context es-serverless-image --style info - ES Serverless image: \`$ES_SERVERLESS_IMAGE\` - - To run this locally: - \`\`\` - node scripts/es serverless --image $ES_SERVERLESS_IMAGE - \`\`\` - EOF + command: .buildkite/scripts/steps/es_serverless/annotate_runtime_parameters.sh - group: "(:kibana: x :elastic:) Trigger Kibana Serverless suite" if: "build.env('SKIP_VERIFICATION') != '1' && build.env('SKIP_VERIFICATION') != 'true'" diff --git a/.buildkite/scripts/steps/es_serverless/annotate_runtime_parameters.sh b/.buildkite/scripts/steps/es_serverless/annotate_runtime_parameters.sh new file mode 100644 index 0000000000000..c3cc571f8a4dc --- /dev/null +++ b/.buildkite/scripts/steps/es_serverless/annotate_runtime_parameters.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -euo pipefail + +KIBANA_GITHUB_URL="https://github.com/elastic/kibana" +ES_SERVERLESS_GITHUB_URL="https://github.com/elastic/elasticsearch-serverless" + +if [[ -z "$ES_SERVERLESS_IMAGE" ]]; then + echo "ES_SERVERLESS_IMAGE is not set" + exit 1 +elif [[ "$ES_SERVERLESS_IMAGE" != *"docker.elastic.co"* ]]; then + echo "ES_SERVERLESS_IMAGE should be a docker.elastic.co image" + exit 1 +fi + +# Pull the target image +if [[ $ES_SERVERLESS_IMAGE != *":git-"* ]]; then + docker pull "$ES_SERVERLESS_IMAGE" + ES_SERVERLESS_VERSION=$(docker inspect --format='{{json .Config.Labels}}' "$ES_SERVERLESS_IMAGE" | jq -r '.["org.opencontainers.image.revision"]' | cut -c1-12) + + IMAGE_WITHOUT_TAG=$(echo "$ES_SERVERLESS_IMAGE" | cut -d: -f1) + ES_SERVERLESS_IMAGE_FULL="${IMAGE_WITHOUT_TAG}:git-${ES_SERVERLESS_VERSION}" +else + ES_SERVERLESS_IMAGE_FULL=$ES_SERVERLESS_IMAGE + ES_SERVERLESS_VERSION=$(echo "$ES_SERVERLESS_IMAGE_FULL" | cut -d: -f2 | cut -d- -f2) +fi + +buildkite-agent annotate --context kibana-commit --style info "Kibana version: $BUILDKITE_BRANCH / [$BUILDKITE_COMMIT]($KIBANA_GITHUB_URL/commit/$BUILDKITE_COMMIT)" +buildkite-agent annotate --context es-serverless-commit --style info "ES Serverless version: [$ES_SERVERLESS_VERSION]($ES_SERVERLESS_GITHUB_URL/commit/$ES_SERVERLESS_VERSION)" + +cat << EOF | buildkite-agent annotate --context es-serverless-image --style info + ES Serverless image: \`${ES_SERVERLESS_IMAGE_FULL}\` + + To run this locally: + \`\`\` + node scripts/es serverless --image $ES_SERVERLESS_IMAGE_FULL + \`\`\` +EOF diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index b3e7bd30ad83e..ec5bbdefdbeac 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -15,6 +15,7 @@ import { detectRunningNodes, maybeCreateDockerNetwork, maybePullDockerImage, + printESImageInfo, resolveDockerCmd, resolveDockerImage, resolveEsArgs, @@ -660,8 +661,14 @@ describe('runServerlessCluster()', () => { await runServerlessCluster(log, { projectType, basePath: baseEsPath }); - // setupDocker execa calls then run three nodes and attach logger - expect(execa.mock.calls).toHaveLength(8); + // docker version (1) + // docker ps (1) + // docker network create (1) + // docker pull (1) + // docker inspect (1) + // docker run (3) + // docker logs (1) + expect(execa.mock.calls).toHaveLength(9); }); test(`should wait for serverless nodes to return 'green' status`, async () => { @@ -795,7 +802,63 @@ describe('runDockerContainer()', () => { test('should resolve', async () => { execa.mockImplementation(() => Promise.resolve({ stdout: '' })); await expect(runDockerContainer(log, {})).resolves.toBeUndefined(); - // setupDocker execa calls then run container - expect(execa.mock.calls).toHaveLength(5); + // docker version (1) + // docker ps (1) + // docker network create (1) + // docker pull (1) + // docker inspect (1) + // docker run (1) + expect(execa.mock.calls).toHaveLength(6); + }); +}); + +describe('printESImageInfo', () => { + beforeEach(() => { + logWriter.messages.length = 0; + }); + + test('should print ES Serverless image info', async () => { + execa.mockImplementation(() => + Promise.resolve({ + stdout: JSON.stringify({ + 'org.opencontainers.image.revision': 'deadbeef12345678', + 'org.opencontainers.image.source': 'https://github.com/elastic/elasticsearch-serverless', + }), + }) + ); + + await printESImageInfo( + log, + 'docker.elastic.co/elasticsearch-ci/elasticsearch-serverless:latest' + ); + + expect(execa.mock.calls).toHaveLength(1); + expect(logWriter.messages[0]).toContain( + `docker.elastic.co/elasticsearch-ci/elasticsearch-serverless:git-deadbeef1234` + ); + expect(logWriter.messages[0]).toContain( + `https://github.com/elastic/elasticsearch-serverless/commit/deadbeef12345678` + ); + }); + + test('should print ES image info', async () => { + execa.mockImplementation(() => + Promise.resolve({ + stdout: JSON.stringify({ + 'org.opencontainers.image.revision': 'deadbeef12345678', + 'org.opencontainers.image.source': 'https://github.com/elastic/elasticsearch', + }), + }) + ); + + await printESImageInfo(log, 'docker.elastic.co/elasticsearch/elasticsearch:8.15-SNAPSHOT'); + + expect(execa.mock.calls).toHaveLength(1); + expect(logWriter.messages[0]).toContain( + `docker.elastic.co/elasticsearch/elasticsearch:8.15-SNAPSHOT` + ); + expect(logWriter.messages[0]).toContain( + `https://github.com/elastic/elasticsearch/commit/deadbeef12345678` + ); }); }); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 60232c97897d2..0e8182920feba 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -26,6 +26,7 @@ import { createMockIdpMetadata, } from '@kbn/mock-idp-utils'; +import { getServerlessImageTag, getCommitUrl } from './extract_image_info'; import { waitForSecurityIndex } from './wait_for_security_index'; import { createCliError } from '../errors'; import { EsClusterExecOptions } from '../cluster_exec_options'; @@ -393,15 +394,29 @@ export async function maybePullDockerImage(log: ToolingLog, image: string) { // inherit is required to show Docker pull output stdio: ['ignore', 'inherit', 'pipe'], }).catch(({ message }) => { - throw createCliError( - `Error pulling image. This is likely an issue authenticating with ${DOCKER_REGISTRY}. + const errorMessage = `Error pulling image. This is likely an issue authenticating with ${DOCKER_REGISTRY}. Visit ${chalk.bold.cyan('https://docker-auth.elastic.co/github_auth')} to login. -${message}` - ); +${message}`; + throw createCliError(errorMessage); }); } +/** + * When we're working with :latest or :latest-verified, it is useful to expand what version they refer to + */ +export async function printESImageInfo(log: ToolingLog, image: string) { + let imageFullName = image; + if (image.includes('serverless')) { + const imageTag = (await getServerlessImageTag(image)) ?? image.split(':').pop() ?? ''; + const imageBase = image.replace(/:.*/, ''); + imageFullName = `${imageBase}:${imageTag}`; + } + + const revisionUrl = await getCommitUrl(image); + log.info(`Using ES image: ${imageFullName} (${revisionUrl})`); +} + export async function detectRunningNodes( log: ToolingLog, options: ServerlessOptions | DockerOptions @@ -445,6 +460,7 @@ async function setupDocker({ await detectRunningNodes(log, options); await maybeCreateDockerNetwork(log); await maybePullDockerImage(log, image); + await printESImageInfo(log, image); } /** diff --git a/packages/kbn-es/src/utils/extract_image_info.ts b/packages/kbn-es/src/utils/extract_image_info.ts new file mode 100644 index 0000000000000..7576ab6ddeff3 --- /dev/null +++ b/packages/kbn-es/src/utils/extract_image_info.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import execa from 'execa'; +import memoize from 'lodash/memoize'; + +export const extractImageInfo = memoize(async (image: string) => { + try { + const { stdout: labelsJson } = await execa( + 'docker', + ['inspect', '--format', '{{json .Config.Labels}}', image], + { + encoding: 'utf8', + } + ); + return JSON.parse(labelsJson); + } catch (e) { + return {}; + } +}); + +export async function getImageVersion(image: string): Promise { + const imageLabels = await extractImageInfo(image); + return imageLabels['org.opencontainers.image.revision'] || null; +} + +export async function getCommitUrl(image: string): Promise { + const imageLabels = await extractImageInfo(image); + const repoSource = imageLabels['org.opencontainers.image.source'] || null; + const revision = imageLabels['org.opencontainers.image.revision'] || null; + + if (!repoSource || !revision) { + return null; + } else { + return `${repoSource}/commit/${revision}`; + } +} + +export async function getServerlessImageTag(image: string): Promise { + const sha = await getImageVersion(image); + if (!sha) { + return null; + } else { + return `git-${sha.slice(0, 12)}`; + } +} diff --git a/packages/kbn-es/src/utils/extract_serverless_image_info.test.ts b/packages/kbn-es/src/utils/extract_serverless_image_info.test.ts new file mode 100644 index 0000000000000..afd9d3e56a29b --- /dev/null +++ b/packages/kbn-es/src/utils/extract_serverless_image_info.test.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + extractImageInfo, + getCommitUrl, + getImageVersion, + getServerlessImageTag, +} from './extract_image_info'; + +jest.mock('execa'); +const execa = jest.requireMock('execa'); + +describe('extractImageInfo', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('calls docker, once, and only once for one image', async () => { + const image = 'nevermind'; + const image2 = 'nevermind2'; + const labelsJson = '{"org.opencontainers.image.revision": "revision"}'; + execa.mockResolvedValue({ stdout: labelsJson }); + + await extractImageInfo(image); + await extractImageInfo(image); + expect(execa).toHaveBeenCalledTimes(1); + + await extractImageInfo(image2); + expect(execa).toHaveBeenCalledTimes(2); + }); + + it('should return image labels as an object', () => { + const image = 'nevermind123'; + const obj = { 'org.opencontainers.image.revision': 'revision', extra: 123 }; + const labelsJson = JSON.stringify(obj); + execa.mockResolvedValue({ stdout: labelsJson }); + + const imageInfo = extractImageInfo(image); + + expect(imageInfo).resolves.toEqual(obj); + }); +}); + +describe('getImageVersion', () => { + it("should return the image's revision", () => { + const image = 'test-image'; + const labels = { 'org.opencontainers.image.revision': 'deadbeef1234' }; + execa.mockResolvedValue({ stdout: JSON.stringify(labels) }); + + const imageVersion = getImageVersion(image); + + expect(imageVersion).resolves.toBe('deadbeef1234'); + }); +}); + +describe('getCommitUrl', () => { + it('should return the commit url', () => { + const image = 'docker.elastic.co/elasticsearch/elasticsearch:7.15.0'; + const labels = { + 'org.opencontainers.image.source': 'https://github.com/elastic/elasticsearch', + 'org.opencontainers.image.revision': 'deadbeef1234', + }; + execa.mockResolvedValue({ stdout: JSON.stringify(labels) }); + + expect(getCommitUrl(image)).resolves.toBe( + 'https://github.com/elastic/elasticsearch/commit/deadbeef1234' + ); + }); +}); + +describe('getServerlessImageTag', () => { + it('should return the image tag', () => { + const image = 'docker.elastic.co/elasticsearch-ci/elasticsearch-serverless:latest'; + const labels = { 'org.opencontainers.image.revision': 'deadbeef12345678' }; + execa.mockResolvedValue({ stdout: JSON.stringify(labels) }); + + const imageTag = getServerlessImageTag(image); + + expect(imageTag).resolves.toBe('git-deadbeef1234'); + }); +});