[Ops] Buildkite job for serverless deployment (elastic#170655)

## Summary
Connected to: elastic/kibana-operations#18
Pre-requisite for:
elastic/kibana-operations#30

You can test the current assistant from the branch:
https://buildkite.com/elastic/kibana-serverless-release-1/builds?branch=buildkite-job-for-deployment
- use `DRY_RUN=1` in the runtime params to not trigger an actual release
:)

This PR creates the contents of a Buildkite job to assist the Kibana
Serverless Release initiation process at the very beginning and lay some
groundwork for further additions to the release management.

At the end of the day, we would like to create a tag deploy@<timestamp>
which will be picked up by another job that listens to these tags:
https://buildkite.com/elastic/kibana-serverless-release. However,
several parts of the preparation for release require manual research,
collecting information about target releases, running scripts, etc.

Any further addition to what would be useful for someone wanting to
start a release could be contained here.

Furthermore, we could also trigger downstream jobs from here. e.g.:
https://buildkite.com/elastic/kibana-serverless-release is currently set
up to listen for a git tag, but we may as well just trigger the job
after we've created a tag.

Check out an example run at:
https://buildkite.com/elastic/kibana-serverless-release-1/builds/72
(visible only if you're a
member of @ elastic/kibana-release-operators) 

Missing features compared to the git action:

- [x] Slack notification about the started deploy
- [x] full "useful links" section

Missing features:
- [x] there's a bit of useful context that should be integrated to the
display of the FTR results (*)
- [x] skip listing and analysis if a commit sha is passed in env


(*) - Currently, we display the next FTR test suite that ran after the
merge of the PR. However, the next FTR that will contain the changes,
and show useful info related to the changeset is ONLY in the FTR that's
ran after the first successful onMerge after the merge commit. Meaning:
if main is failing when the change is merged, an FTR suite won't pick up
the change right after.

---------

Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: Thomas Watson <[email protected]>
Co-authored-by: Thomas Watson <[email protected]>

.buildkite/pipeline-utils/buildkite/client.ts

@@ -7,24 +7,31 @@
  */
 
 import axios, { AxiosInstance } from 'axios';
-import { execSync } from 'child_process';
+import { execSync, ExecSyncOptions } from 'child_process';
 import { dump } from 'js-yaml';
 import { parseLinkHeader } from './parse_link_header';
 import { Artifact } from './types/artifact';
 import { Build, BuildStatus } from './types/build';
 import { Job, JobState } from './types/job';
 
+type ExecType =
+  | ((command: string, execOpts: ExecSyncOptions) => Buffer | null)
+  | ((command: string, execOpts: ExecSyncOptions) => string | null);
+
 export interface BuildkiteClientConfig {
   baseUrl?: string;
   token?: string;
+  exec?: ExecType;
 }
 
 export interface BuildkiteGroup {
   group: string;
   steps: BuildkiteStep[];
 }
 
-export interface BuildkiteStep {
+export type BuildkiteStep = BuildkiteCommandStep | BuildkiteInputStep;
+
+export interface BuildkiteCommandStep {
   command: string;
   label: string;
   parallelism?: number;
@@ -43,6 +50,50 @@ export interface BuildkiteStep {
   env?: { [key: string]: string };
 }
 
+interface BuildkiteInputTextField {
+  text: string;
+  key: string;
+  hint?: string;
+  required?: boolean;
+  default?: string;
+}
+
+interface BuildkiteInputSelectField {
+  select: string;
+  key: string;
+  hint?: string;
+  required?: boolean;
+  default?: string;
+  multiple?: boolean;
+  options: Array<{
+    label: string;
+    value: string;
+  }>;
+}
+
+export interface BuildkiteInputStep {
+  input: string;
+  prompt?: string;
+  fields: Array<BuildkiteInputTextField | BuildkiteInputSelectField>;
+  if?: string;
+  allow_dependency_failure?: boolean;
+  branches?: string;
+  parallelism?: number;
+  agents?: {
+    queue: string;
+  };
+  timeout_in_minutes?: number;
+  key?: string;
+  depends_on?: string | string[];
+  retry?: {
+    automatic: Array<{
+      exit_status: string;
+      limit: number;
+    }>;
+  };
+  env?: { [key: string]: string };
+}
+
 export interface BuildkiteTriggerBuildParams {
   commit: string;
   branch: string;
@@ -61,6 +112,7 @@ export interface BuildkiteTriggerBuildParams {
 
 export class BuildkiteClient {
   http: AxiosInstance;
+  exec: ExecType;
 
   constructor(config: BuildkiteClientConfig = {}) {
     const BUILDKITE_BASE_URL =
@@ -78,6 +130,8 @@ export class BuildkiteClient {
       },
     });
 
+    this.exec = config.exec ?? execSync;
+
     // this.agentHttp = axios.create({
     //   baseURL: BUILDKITE_AGENT_BASE_URL,
     //   headers: {
@@ -97,6 +151,32 @@ export class BuildkiteClient {
     return resp.data as Build;
   };
 
+  getBuildsAfterDate = async (
+    pipelineSlug: string,
+    date: string,
+    numberOfBuilds: number
+  ): Promise<Build[]> => {
+    const response = await this.http.get(
+      `v2/organizations/elastic/pipelines/${pipelineSlug}/builds?created_from=${date}&per_page=${numberOfBuilds}`
+    );
+    return response.data as Build[];
+  };
+
+  getBuildForCommit = async (pipelineSlug: string, commit: string): Promise<Build | null> => {
+    if (commit.length !== 40) {
+      throw new Error(`Invalid commit hash: ${commit}, this endpoint works with full SHAs only`);
+    }
+
+    const response = await this.http.get(
+      `v2/organizations/elastic/pipelines/${pipelineSlug}/builds?commit=${commit}`
+    );
+    const builds = response.data as Build[];
+    if (builds.length === 0) {
+      return null;
+    }
+    return builds[0];
+  };
+
   getCurrentBuild = (includeRetriedJobs = false) => {
     if (!process.env.BUILDKITE_PIPELINE_SLUG || !process.env.BUILDKITE_BUILD_NUMBER) {
       throw new Error(
@@ -235,31 +315,46 @@ export class BuildkiteClient {
   };
 
   setMetadata = (key: string, value: string) => {
-    execSync(`buildkite-agent meta-data set '${key}'`, {
+    this.exec(`buildkite-agent meta-data set '${key}'`, {
       input: value,
       stdio: ['pipe', 'inherit', 'inherit'],
     });
   };
 
+  getMetadata(key: string, defaultValue: string | null = null): string | null {
+    try {
+      const stdout = this.exec(`buildkite-agent meta-data get '${key}'`, {
+        stdio: ['pipe'],
+      });
+      return stdout?.toString().trim() || defaultValue;
+    } catch (e) {
+      if (e.message.includes('404 Not Found')) {
+        return defaultValue;
+      } else {
+        throw e;
+      }
+    }
+  }
+
   setAnnotation = (
     context: string,
     style: 'info' | 'success' | 'warning' | 'error',
     value: string
   ) => {
-    execSync(`buildkite-agent annotate --context '${context}' --style '${style}'`, {
+    this.exec(`buildkite-agent annotate --context '${context}' --style '${style}'`, {
       input: value,
       stdio: ['pipe', 'inherit', 'inherit'],
     });
   };
 
   uploadArtifacts = (pattern: string) => {
-    execSync(`buildkite-agent artifact upload '${pattern}'`, {
+    this.exec(`buildkite-agent artifact upload '${pattern}'`, {
       stdio: ['ignore', 'inherit', 'inherit'],
     });
   };
 
   uploadSteps = (steps: Array<BuildkiteStep | BuildkiteGroup>) => {
-    execSync(`buildkite-agent pipeline upload`, {
+    this.exec(`buildkite-agent pipeline upload`, {
       input: dump({ steps }),
       stdio: ['pipe', 'inherit', 'inherit'],
     });

.buildkite/pipeline-utils/github/github.ts

@@ -91,3 +91,7 @@ export const doAnyChangesMatch = async (
 
   return anyFilesMatchRequired;
 };
+
+export function getGithubClient() {
+  return github;
+}

.buildkite/pipeline-utils/index.ts

@@ -10,3 +10,4 @@ export * from './buildkite';
 export * as CiStats from './ci-stats';
 export * from './github';
 export * as TestFailures from './test-failures';
+export * from './utils';

.buildkite/pipeline-utils/utils.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { execSync } from 'child_process';
+
+const getKibanaDir = (() => {
+  let kibanaDir: string | undefined;
+  return () => {
+    if (!kibanaDir) {
+      kibanaDir = execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' })
+        .toString()
+        .trim();
+    }
+
+    return kibanaDir;
+  };
+})();
+
+export { getKibanaDir };

.buildkite/pipelines/serverless_deployment/create_deployment_tag.yml

@@ -0,0 +1,33 @@
+## Creates deploy@<timestamp> tag on Kibana
+
+agents:
+ queue: kibana-default
+
+steps:
+  - label: "List potential commits"
+    commands:
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state initialize
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state collect_commits
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/list_commit_candidates.ts 25
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state wait_for_selection
+    key: select_commit
+
+  - wait: ~
+
+  - label: "Collect commit info"
+    commands:
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state collect_commit_info
+      - bash .buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.sh
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state wait_for_confirmation
+    key: collect_data
+    depends_on: select_commit
+
+  - wait: ~
+
+  - label: ":ship: Create Deploy Tag"
+    commands:
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state create_deploy_tag
+      - bash .buildkite/scripts/serverless/create_deploy_tag/create_deploy_tag.sh
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state tag_created
+    env:
+      DRY_RUN: $DRY_RUN

.buildkite/scripts/lifecycle/pre_command.sh

@@ -139,6 +139,9 @@ export SYNTHETICS_REMOTE_KIBANA_PASSWORD
 SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(retry 5 5 vault read -field=url secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"}
 export SYNTHETICS_REMOTE_KIBANA_URL
 
+DEPLOY_TAGGER_SLACK_WEBHOOK_URL=${DEPLOY_TAGGER_SLACK_WEBHOOK_URL:-"$(retry 5 5 vault read -field=DEPLOY_TAGGER_SLACK_WEBHOOK_URL secret/kibana-issues/dev/kibana-serverless-release-tools)"}
+export DEPLOY_TAGGER_SLACK_WEBHOOK_URL
+
 # Setup Failed Test Reporter Elasticsearch credentials
 {
   TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)

.buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# SO migration comparison lives in the Kibana dev app code, needs bootstrapping
+.buildkite/scripts/bootstrap.sh
+
+echo "--- Collecting commit info"
+ts-node .buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.ts
+
+cat << EOF | buildkite-agent pipeline upload
+  steps:
+    - block: "Confirm deployment"
+      prompt: "Are you sure you want to deploy to production? (dry run: ${DRY_RUN:-false})"
+      depends_on: collect_data
+EOF

.buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { COMMIT_INFO_CTX, exec } from './shared';
+import {
+  toGitCommitExtract,
+  getCurrentQARelease,
+  getSelectedCommitHash,
+  getCommitByHash,
+  makeCommitInfoHtml,
+} from './info_sections/commit_info';
+import {
+  getArtifactBuild,
+  getOnMergePRBuild,
+  getQAFBuildContainingCommit,
+  makeBuildkiteBuildInfoHtml,
+} from './info_sections/build_info';
+import {
+  compareSOSnapshots,
+  makeSOComparisonBlockHtml,
+  makeSOComparisonErrorHtml,
+} from './info_sections/so_snapshot_comparison';
+import { makeUsefulLinksHtml } from './info_sections/useful_links';
+
+async function main() {
+  const previousSha = await getCurrentQARelease();
+  const selectedSha = getSelectedCommitHash();
+
+  // Current commit info
+  const previousCommit = await getCommitByHash(previousSha);
+  const previousCommitInfo = toGitCommitExtract(previousCommit);
+  addBuildkiteInfoSection(makeCommitInfoHtml('Current commit on QA:', previousCommitInfo));
+
+  // Target commit info
+  const selectedCommit = await getCommitByHash(selectedSha);
+  const selectedCommitInfo = toGitCommitExtract(selectedCommit);
+  addBuildkiteInfoSection(makeCommitInfoHtml('Target commit to deploy:', selectedCommitInfo));
+
+  // Buildkite build info
+  const buildkiteBuild = await getOnMergePRBuild(selectedSha);
+  const nextBuildContainingCommit = await getQAFBuildContainingCommit(
+    selectedSha,
+    selectedCommitInfo.date!
+  );
+  const artifactBuild = await getArtifactBuild(selectedSha);
+  addBuildkiteInfoSection(
+    makeBuildkiteBuildInfoHtml('Relevant build info:', {
+      'Merge build': buildkiteBuild,
+      'Artifact container build': artifactBuild,
+      'Next QAF test build containing this commit': nextBuildContainingCommit,
+    })
+  );
+
+  // Save Object migration comparison
+  const comparisonResult = compareSOSnapshots(previousSha, selectedSha);
+  if (comparisonResult) {
+    addBuildkiteInfoSection(makeSOComparisonBlockHtml(comparisonResult));
+  } else {
+    addBuildkiteInfoSection(makeSOComparisonErrorHtml());
+  }
+
+  // Useful links
+  addBuildkiteInfoSection(
+    makeUsefulLinksHtml('Useful links:', {
+      previousCommitHash: previousSha,
+      selectedCommitHash: selectedSha,
+    })
+  );
+}
+
+function addBuildkiteInfoSection(html: string) {
+  exec(`buildkite-agent annotate --append --style 'info' --context '${COMMIT_INFO_CTX}'`, {
+    input: html + '<br />',
+  });
+}
+
+main()
+  .then(() => {
+    console.log('Commit-related information added.');
+  })
+  .catch((error) => {
+    console.error(error);
+    process.exit(1);
+  })
+  .finally(() => {
+    // When running locally, we can see what calls were made to execSync to debug
+    if (!process.env.CI) {
+      // @ts-ignore
+      console.log(exec.calls);
+    }
+  });