Skip to content

Commit

Permalink
Merge branch 'master' into reporting/feature-control-example
Browse files Browse the repository at this point in the history
  • Loading branch information
kibanamachine authored Dec 22, 2020
2 parents a2ad912 + 42c1414 commit f7d8a70
Show file tree
Hide file tree
Showing 620 changed files with 7,987 additions and 4,619 deletions.
2 changes: 1 addition & 1 deletion .ci/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
# If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts

ARG NODE_VERSION=14.15.2
ARG NODE_VERSION=14.15.3

FROM node:${NODE_VERSION} AS base

Expand Down
2 changes: 1 addition & 1 deletion .ci/teamcity/bootstrap.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ source "$(dirname "${0}")/util.sh"
tc_start_block "Bootstrap"

tc_start_block "yarn install and kbn bootstrap"
verify_no_git_changes yarn kbn bootstrap --prefer-offline
verify_no_git_changes yarn kbn bootstrap
tc_end_block "yarn install and kbn bootstrap"

tc_start_block "build kbn-pm"
Expand Down
3 changes: 2 additions & 1 deletion .ci/teamcity/checks/bundle_limits.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@ set -euo pipefail

source "$(dirname "${0}")/../util.sh"

node scripts/build_kibana_platform_plugins --validate-limits
checks-reporter-with-killswitch "Check Bundle Limits" \
node scripts/build_kibana_platform_plugins --validate-limits
13 changes: 13 additions & 0 deletions .ci/teamcity/checks/commit.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/usr/bin/env bash

set -euo pipefail

source "$(dirname "${0}")/../util.sh"

# Runs pre-commit hook script for the files touched in the last commit.
# That way we can ensure a set of quick commit checks earlier as we removed
# the pre-commit hook installation by default.
# If files are more than 200 we will skip it and just use
# the further ci steps that already check linting and file casing for the entire repo.
checks-reporter-with-killswitch "Quick commit checks" \
"$(dirname "${0}")/commit_check_runner.sh"
9 changes: 9 additions & 0 deletions .ci/teamcity/checks/commit_check_runner.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/usr/bin/env bash

echo "!!!!!!!! ATTENTION !!!!!!!!
That check is intended to provide earlier CI feedback after we remove the automatic install for the local pre-commit hook.
If you want, you can still manually install the pre-commit hook locally by running 'node scripts/register_git_hook locally'
!!!!!!!!!!!!!!!!!!!!!!!!!!!
"

node scripts/precommit_hook.js --ref HEAD~1..HEAD --max-files 200 --verbose
8 changes: 8 additions & 0 deletions .ci/teamcity/checks/jest_configs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

set -euo pipefail

source "$(dirname "${0}")/../util.sh"

checks-reporter-with-killswitch "Check Jest Configs" \
node scripts/check_jest_configs
8 changes: 8 additions & 0 deletions .ci/teamcity/checks/plugins_with_circular_deps.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

set -euo pipefail

source "$(dirname "${0}")/../util.sh"

checks-reporter-with-killswitch "Check Plugins With Circular Dependencies" \
node scripts/find_plugins_with_circular_deps
21 changes: 18 additions & 3 deletions .ci/teamcity/oss/plugin_functional.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,21 @@ if [[ ! -d "target" ]]; then
fi
cd -

./test/scripts/test/plugin_functional.sh
./test/scripts/test/example_functional.sh
./test/scripts/test/interpreter_functional.sh
checks-reporter-with-killswitch "Plugin Functional Tests" \
node scripts/functional_tests \
--config test/plugin_functional/config.ts \
--bail \
--debug

checks-reporter-with-killswitch "Example Functional Tests" \
node scripts/functional_tests \
--config test/examples/config.js \
--bail \
--debug

checks-reporter-with-killswitch "Interpreter Functional Tests" \
node scripts/functional_tests \
--config test/interpreter_functional/config.ts \
--bail \
--debug \
--kibana-install-dir "$KIBANA_INSTALL_DIR"
3 changes: 3 additions & 0 deletions .ci/teamcity/setup_env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,14 @@ tc_set_env FORCE_COLOR 1
tc_set_env TEST_BROWSER_HEADLESS 1

tc_set_env ELASTIC_APM_ENVIRONMENT ci
tc_set_env ELASTIC_APM_TRANSACTION_SAMPLE_RATE 0.1

if [[ "${KIBANA_CI_REPORTER_KEY_BASE64-}" ]]; then
tc_set_env KIBANA_CI_REPORTER_KEY "$(echo "$KIBANA_CI_REPORTER_KEY_BASE64" | base64 -d)"
fi

if is_pr; then
tc_set_env ELASTIC_APM_ACTIVE false
tc_set_env CHECKS_REPORTER_ACTIVE true

# These can be removed once we're not supporting Jenkins and TeamCity at the same time
Expand All @@ -39,6 +41,7 @@ if is_pr; then
tc_set_env ghprbActualCommit "$GITHUB_PR_TRIGGERED_SHA"
tc_set_env BUILD_URL "$TEAMCITY_BUILD_URL"
else
tc_set_env ELASTIC_APM_ACTIVE true
tc_set_env CHECKS_REPORTER_ACTIVE false
fi

Expand Down
2 changes: 1 addition & 1 deletion .node-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
14.15.2
14.15.3
2 changes: 1 addition & 1 deletion .nvmrc
Original file line number Diff line number Diff line change
@@ -1 +1 @@
14.15.2
14.15.3
17 changes: 17 additions & 0 deletions .teamcity/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,25 @@
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>teamcity</id>
<url>https://artifactory.elstc.co/artifactory/teamcity</url>
<snapshots>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
</snapshots>
</repository>
</repositories>

<pluginRepositories>
<pluginRepository>
<id>JetBrains</id>
<url>https://download.jetbrains.com/teamcity-repository</url>
</pluginRepository>
<pluginRepository>
<id>teamcity</id>
<url>https://artifactory.elstc.co/artifactory/teamcity</url>
</pluginRepository>
</pluginRepositories>

<build>
Expand Down Expand Up @@ -124,5 +136,10 @@
<artifactId>junit</artifactId>
<version>4.13</version>
</dependency>
<dependency>
<groupId>co.elastic.teamcity</groupId>
<artifactId>teamcity-common</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>
2 changes: 1 addition & 1 deletion .teamcity/settings.kts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import jetbrains.buildServer.configs.kotlin.v2019_2.*
import projects.Kibana
import projects.KibanaConfiguration

version = "2020.1"
version = "2020.2"

val config = KibanaConfiguration {
agentNetwork = DslContext.getParameter("agentNetwork", "teamcity")
Expand Down
28 changes: 28 additions & 0 deletions .teamcity/src/Agents.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import co.elastic.teamcity.common.GoogleCloudAgent
import co.elastic.teamcity.common.GoogleCloudAgentDiskType
import co.elastic.teamcity.common.GoogleCloudProfile

private val sizes = listOf("2", "4", "8", "16")

val StandardAgents = sizes.map { size -> size to GoogleCloudAgent {
sourceImageFamily = "elastic-kibana-ci-ubuntu-1804-lts"
agentPrefix = "kibana-standard-$size-"
machineType = "n2-standard-$size"
diskSizeGb = 75
diskType = GoogleCloudAgentDiskType.SSD
} }.toMap()

val BuildAgent = GoogleCloudAgent {
sourceImageFamily = "elastic-kibana-ci-ubuntu-1804-lts"
agentPrefix = "kibana-c2-16-"
machineType = "c2-standard-16"
diskSizeGb = 250
diskType = GoogleCloudAgentDiskType.SSD
}

val CloudProfile = GoogleCloudProfile {
accessKeyId = "447fdd4d-7129-46b7-9822-2e57658c7422"

agents(StandardAgents)
agent(BuildAgent)
}
38 changes: 2 additions & 36 deletions .teamcity/src/Extensions.kt
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
import co.elastic.teamcity.common.requireAgent
import jetbrains.buildServer.configs.kotlin.v2019_2.*
import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.notifications
import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.ScriptBuildStep
import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script
import jetbrains.buildServer.configs.kotlin.v2019_2.ui.insert
import projects.kibanaConfiguration

fun BuildFeatures.junit(dirs: String = "target/**/TEST-*.xml") {
feature {
Expand All @@ -13,40 +11,8 @@ fun BuildFeatures.junit(dirs: String = "target/**/TEST-*.xml") {
}
}

fun ProjectFeatures.kibanaAgent(init: ProjectFeature.() -> Unit) {
feature {
type = "CloudImage"
param("network", kibanaConfiguration.agentNetwork)
param("subnet", kibanaConfiguration.agentSubnet)
param("growingId", "true")
param("agent_pool_id", "-2")
param("preemptible", "false")
param("sourceProject", "elastic-images-prod")
param("sourceImageFamily", "elastic-kibana-ci-ubuntu-1804-lts")
param("zone", "us-central1-a")
param("profileId", "kibana")
param("diskType", "pd-ssd")
param("machineCustom", "false")
param("maxInstances", "200")
param("imageType", "ImageFamily")
param("diskSizeGb", "75") // TODO
init()
}
}

fun ProjectFeatures.kibanaAgent(size: String, init: ProjectFeature.() -> Unit = {}) {
kibanaAgent {
id = "KIBANA_STANDARD_$size"
param("source-id", "kibana-standard-$size-")
param("machineType", "n2-standard-$size")
init()
}
}

fun BuildType.kibanaAgent(size: String) {
requirements {
startsWith("teamcity.agent.name", "kibana-standard-$size-", "RQ_AGENT_NAME")
}
requireAgent(StandardAgents[size]!!)
}

fun BuildType.kibanaAgent(size: Int) {
Expand Down
6 changes: 4 additions & 2 deletions .teamcity/src/builds/Checks.kt
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,18 @@ object Checks : BuildType({
kibanaAgent(4)

val checkScripts = mapOf(
"Quick Commit Checks" to ".ci/teamcity/checks/commit.sh",
"Check Telemetry Schema" to ".ci/teamcity/checks/telemetry.sh",
"Check TypeScript Projects" to ".ci/teamcity/checks/ts_projects.sh",
"Check File Casing" to ".ci/teamcity/checks/file_casing.sh",
"Check Licenses" to ".ci/teamcity/checks/licenses.sh",
"Verify NOTICE" to ".ci/teamcity/checks/verify_notice.sh",
"Test Hardening" to ".ci/teamcity/checks/test_hardening.sh",
"Check Types" to ".ci/teamcity/checks/type_check.sh",
"Check Jest Configs" to ".ci/teamcity/checks/jest_configs.sh",
"Check Doc API Changes" to ".ci/teamcity/checks/doc_api_changes.sh",
"Check Bundle Limits" to ".ci/teamcity/checks/bundle_limits.sh",
"Check i18n" to ".ci/teamcity/checks/i18n.sh"
"Check i18n" to ".ci/teamcity/checks/i18n.sh",
"Check Plugins With Circular Dependencies" to ".ci/teamcity/checks/plugins_with_circular_deps.sh"
)

steps {
Expand Down
4 changes: 4 additions & 0 deletions .teamcity/src/builds/default/DefaultCiGroup.kt
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package builds.default

import StandardAgents
import co.elastic.teamcity.common.requireAgent
import jetbrains.buildServer.configs.kotlin.v2019_2.*
import runbld

Expand All @@ -11,5 +13,7 @@ class DefaultCiGroup(val ciGroup: Int = 0, init: BuildType.() -> Unit = {}) : De
runbld("Default CI Group $ciGroup", "./.ci/teamcity/default/ci_group.sh $ciGroup")
}

requireAgent(StandardAgents["4"]!!)

init()
})
2 changes: 1 addition & 1 deletion .teamcity/src/builds/default/DefaultCiGroups.kt
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package builds.default
import dependsOn
import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType

const val DEFAULT_CI_GROUP_COUNT = 10
const val DEFAULT_CI_GROUP_COUNT = 11
val defaultCiGroups = (1..DEFAULT_CI_GROUP_COUNT).map { DefaultCiGroup(it) }

object DefaultCiGroups : BuildType({
Expand Down
4 changes: 2 additions & 2 deletions .teamcity/src/builds/es_snapshots/Verify.kt
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import builds.default.defaultCiGroups
import builds.oss.OssBuild
import builds.oss.OssPluginFunctional
import builds.oss.ossCiGroups
import builds.test.ApiServerIntegration
import builds.oss.OssApiServerIntegration
import builds.test.JestIntegration
import dependsOn
import jetbrains.buildServer.configs.kotlin.v2019_2.*
Expand Down Expand Up @@ -49,7 +49,7 @@ val defaultBuildsToClone = listOf(
val defaultCloned = defaultBuildsToClone.map { cloneForVerify(it) }

val integrationsBuildsToClone = listOf(
ApiServerIntegration,
OssApiServerIntegration,
JestIntegration
)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
package builds.test
package builds.oss

import addTestSettings
import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
import runbld

object ApiServerIntegration : BuildType({
object OssApiServerIntegration : OssFunctionalBase({
name = "API/Server Integration"
description = "Executes API and Server Integration Tests"

steps {
runbld("API Integration", "./.ci/teamcity/oss/api_integration.sh")
runbld("Server Integration", "./.ci/teamcity/oss/server_integration.sh")
}

addTestSettings()
})
3 changes: 2 additions & 1 deletion .teamcity/src/builds/test/AllTests.kt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package builds.test

import builds.oss.OssApiServerIntegration
import dependsOn
import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType

Expand All @@ -8,5 +9,5 @@ object AllTests : BuildType({
description = "All Non-Functional Tests"
type = Type.COMPOSITE

dependsOn(QuickTests, Jest, XPackJest, JestIntegration, ApiServerIntegration)
dependsOn(QuickTests, Jest, XPackJest, JestIntegration, OssApiServerIntegration)
})
2 changes: 1 addition & 1 deletion .teamcity/src/builds/test/QuickTests.kt
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ object QuickTests : BuildType({
kibanaAgent(2)

val testScripts = mapOf(
"Test Hardening" to ".ci/teamcity/checkes/test_hardening.sh",
"Test Hardening" to ".ci/teamcity/checks/test_hardening.sh",
"Test Projects" to ".ci/teamcity/tests/test_projects.sh",
"Mocha Tests" to ".ci/teamcity/tests/mocha.sh"
)
Expand Down
Loading

0 comments on commit f7d8a70

Please sign in to comment.