From 48959e769cb29b02e8c49d68fb2c7f9f8c3418d0 Mon Sep 17 00:00:00 2001
From: Thom Heymann <190132+thomheymann@users.noreply.github.com>
Date: Thu, 24 Oct 2024 15:43:01 +0100
Subject: [PATCH] [Observability Onboarding] Display next steps (#197179)
## Summary
Display next steps after the auto-detect script completes.
## Screenshot
---
.../services/epm/package_service.mock.ts | 2 +-
.../server/services/epm/package_service.ts | 13 ++--
.../public/assets/auto_detect.sh | 35 ++++++---
.../public/assets/integrations.conf | 26 +++----
.../server/routes/flow/route.ts | 72 ++++++-------------
.../server/routes/types.ts | 2 +-
6 files changed, 67 insertions(+), 83 deletions(-)
diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts
index 39d0451687de5..eeaa80b0c9449 100644
--- a/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts
+++ b/x-pack/plugins/fleet/server/services/epm/package_service.mock.ts
@@ -16,7 +16,7 @@ const createClientMock = (): jest.Mocked => ({
installCustomIntegration: jest.fn(),
fetchFindLatestPackage: jest.fn(),
readBundledPackage: jest.fn(),
- getAgentPolicyInputs: jest.fn(),
+ getAgentPolicyConfigYAML: jest.fn(),
getPackage: jest.fn(),
getPackageFieldsMetadata: jest.fn(),
getPackages: jest.fn(),
diff --git a/x-pack/plugins/fleet/server/services/epm/package_service.ts b/x-pack/plugins/fleet/server/services/epm/package_service.ts
index 1911ed14a7c80..661475dfadc09 100644
--- a/x-pack/plugins/fleet/server/services/epm/package_service.ts
+++ b/x-pack/plugins/fleet/server/services/epm/package_service.ts
@@ -28,7 +28,6 @@ import type {
InstallablePackage,
Installation,
RegistryPackage,
- TemplateAgentPolicyInput,
} from '../../types';
import type { FleetAuthzRouteConfig } from '../security/types';
@@ -116,12 +115,12 @@ export interface PackageClient {
prerelease?: false;
}): Promise;
- getAgentPolicyInputs(
+ getAgentPolicyConfigYAML(
pkgName: string,
pkgVersion?: string,
prerelease?: false,
ignoreUnverified?: boolean
- ): Promise;
+ ): Promise;
reinstallEsAssets(
packageInfo: InstallablePackage,
@@ -284,7 +283,7 @@ class PackageClientImpl implements PackageClient {
return generatePackageInfoFromArchiveBuffer(archiveBuffer, 'application/zip');
}
- public async getAgentPolicyInputs(
+ public async getAgentPolicyConfigYAML(
pkgName: string,
pkgVersion?: string,
prerelease?: false,
@@ -298,16 +297,14 @@ class PackageClientImpl implements PackageClient {
pkgVersion = pkg.version;
}
- const { inputs } = await getTemplateInputs(
+ return getTemplateInputs(
this.internalSoClient,
pkgName,
pkgVersion,
- 'json',
+ 'yml',
prerelease,
ignoreUnverified
);
-
- return inputs;
}
public async getPackage(
diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh
index ebdcdeb0d81dc..c315ef483d9d6 100755
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh
@@ -105,6 +105,7 @@ elastic_agent_config_path="/opt/Elastic/Agent/elastic-agent.yml"
elastic_agent_tmp_config_path="/tmp/elastic-agent-config.tar"
integration_names=()
integration_titles=()
+config_files_with_password=()
OS="$(uname)"
ARCH="$(uname -m)"
@@ -155,7 +156,7 @@ download_elastic_agent() {
curl -L -O "$download_url" --silent --fail
if [ "$?" -eq 0 ]; then
- printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz"
+ printf "\e[32;1m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz"
update_step_progress "ea-download" "complete"
else
update_step_progress "ea-download" "danger" "Failed to download Elastic Agent, see script output for error."
@@ -167,7 +168,7 @@ extract_elastic_agent() {
tar -xzf "${elastic_agent_artifact_name}.tar.gz"
if [ "$?" -eq 0 ]; then
- printf "\e[1;32m✓\e[0m %s\n" "Archive extracted"
+ printf "\e[32;1m✓\e[0m %s\n" "Archive extracted"
update_step_progress "ea-extract" "complete"
else
update_step_progress "ea-extract" "danger" "Failed to extract Elastic Agent, see script output for error."
@@ -179,7 +180,7 @@ install_elastic_agent() {
"./${elastic_agent_artifact_name}/elastic-agent" install -f -n >/dev/null
if [ "$?" -eq 0 ]; then
- printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")"
+ printf "\e[32;1m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")"
update_step_progress "ea-install" "complete"
else
update_step_progress "ea-install" "danger" "Failed to install Elastic Agent, see script output for error."
@@ -224,7 +225,7 @@ ensure_elastic_agent_healthy() {
backup_elastic_agent_config() {
if [ -f "$elastic_agent_config_path" ]; then
- echo -e "\nExisting config found at $elastic_agent_config_path"
+ printf "\n%s \e[36m%s\e[0m\n" "Existing config found at" "$elastic_agent_config_path"
printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Create backup and continue installation?" "[Y/n] (default: Yes): "
read confirmation_reply
@@ -241,7 +242,7 @@ backup_elastic_agent_config() {
fi
if [ "$?" -eq 0 ]; then
- printf "\n\e[1;32m✓\e[0m %s\n" "Backup saved to $backup_path"
+ printf "\n\e[32;1m✓\e[0m %s \e[36m%s\e[0m\n" "Backup saved to" "$backup_path"
else
update_step_progress "ea-config" "warning" "Failed to backup existing configuration"
fail "Failed to backup existing config - Try manually creating a backup or delete your existing config before re-running this script"
@@ -278,7 +279,7 @@ install_integrations() {
--output "$elastic_agent_tmp_config_path"
if [ "$?" -eq 0 ]; then
- printf "\n\e[1;32m✓\e[0m %s\n" "Integrations installed"
+ printf "\n\e[32;1m✓\e[0m %s\n" "Integrations installed"
else
update_step_progress "ea-config" "warning" "Failed to install integrations"
fail "Failed to install integrations"
@@ -297,10 +298,15 @@ apply_elastic_agent_config() {
# Replace placeholder with the Ingest API key
sed -i='' "s/\${API_KEY}/$decoded_ingest_api_key/" "$elastic_agent_config_path"
if [ "$?" -eq 0 ]; then
- printf "\e[1;32m✓\e[0m %s\n" "Config written to:"
- tar --list --file "$elastic_agent_tmp_config_path" | grep '\.yml$' | while read -r file; do
- echo " - $(dirname "$elastic_agent_config_path")/$file"
- done
+ printf "\e[32;1m✓\e[0m %s\n" "Config files written to:"
+ while IFS= read -r file; do
+ local path="$(dirname "$elastic_agent_config_path")/$file"
+ printf " \e[36m%s\e[0m\n" "$path"
+ grep '' "$path" >/dev/null
+ if [ "$?" -eq 0 ]; then
+ config_files_with_password+=("$path")
+ fi
+ done < <(tar --list --file "$elastic_agent_tmp_config_path" | grep '\.yml$')
update_step_progress "ea-config" "complete"
else
@@ -585,4 +591,11 @@ printf "\n\e[1m%s\e[0m\n" "Waiting for healthy status..."
wait_for_elastic_agent_status
ensure_elastic_agent_healthy
-printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running. You can now go back to Kibana and check for incoming logs."
+printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running!"
+
+printf "\n\e[1m%s\e[0m\n" "Next steps:"
+printf "\n• %s\n" "Go back to Kibana and check for incoming data"
+for path in "${config_files_with_password[@]}"; do
+ printf "\n• %s:\n \e[36m%s\e[0m\n" "Collect $(known_integration_title "$(basename "${path%.yml}")") metrics by adding your username and password to" "$path"
+done
+printf "\n• %s:\n \e[36;4m%s\e[0m\n" "For information on other standalone integration setups, visit" "https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html"
diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf
index e6455a9170c86..0b197bef30f7d 100644
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf
@@ -1,14 +1,14 @@
[system]
-title=System Logs And Metrics
+title=System
[nginx]
-title=Nginx Logs
+title=Nginx
patterns=
/var/log/nginx/access.log*
/var/log/nginx/error.log*
[apache]
-title=Apache Logs
+title=Apache
patterns=
/var/log/apache2/access.log*
/var/log/apache2/other_vhosts_access.log*
@@ -17,13 +17,13 @@ patterns=
/var/log/httpd/error_log*
[docker]
-title=Docker Container Logs
+title=Docker
patterns=
/var/lib/docker/containers/*/*-json.log
/var/run/docker.sock
[mysql]
-title=MySQL Logs
+title=MySQL
patterns=
/var/log/mysql/*error.log*
/var/log/mysqld.log*
@@ -31,7 +31,7 @@ patterns=
/var/lib/mysql/*-slow.log*
[postgresql]
-title=PostgreSQL Logs
+title=PostgreSQL
patterns=
/var/log/postgresql/postgresql-*-*.log*
/*/postgresql-logs/*.log
@@ -39,26 +39,26 @@ patterns=
/var/log/postgresql/postgresql-*-*.csv*
[redis]
-title=Redis Logs
+title=Redis
patterns=
/var/log/redis/redis-server.log*
/etc/redis/redis.conf
[haproxy]
-title=HAProxy Logs
+title=HAProxy
patterns=
/var/log/haproxy.log
/etc/haproxy/haproxy.cfg
[rabbitmq]
-title=RabbitMQ Logs
+title=RabbitMQ
patterns=
/var/log/rabbitmq/rabbit@*.log
/etc/rabbitmq/rabbitmq.conf
/etc/rabbitmq/rabbitmq.config
[kafka]
-title=Kafka Logs
+title=Kafka
patterns=
/var/log/kafka/server.log
/etc/kafka/server.properties
@@ -68,19 +68,19 @@ patterns=
/*/logs/kafka-*.log*
[mongodb]
-title=MongoDB Logs
+title=MongoDB
patterns=
/var/log/mongodb/mongod.log
[apache_tomcat]
-title=Apache Tomcat Logs
+title=Apache Tomcat
patterns=
/opt/tomcat/logs/localhost_access_log.*.txt
/opt/tomcat/logs/catalina.*.log
/opt/tomcat/logs/localhost.*.log
[prometheus]
-title=Prometheus Server overview
+title=Prometheus
patterns=
/var/log/prometheus/prometheus.log
/etc/prometheus/prometheus.yml
\ No newline at end of file
diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts
index 229ce3bf252d0..d6575f8751c4a 100644
--- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts
+++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts
@@ -263,8 +263,8 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
*
* The request format is TSV (tab-separated values) to simplify parsing in bash.
*
- * The response format is either a YAML file or a tar archive containing the Elastic Agent
- * configuration, depending on the `Accept` header.
+ * The response format is a tar archive containing the Elastic Agent configuration, depending on the
+ * `Accept` header.
*
* Errors during installation are ignore unless all integrations fail to install. When that happens
* a 500 Internal Server Error is returned with the first error message.
@@ -348,7 +348,7 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
}
return acc;
}, []);
- // Errors during installation are ignore unless all integrations fail to install. When that happens
+ // Errors during installation are ignored unless all integrations fail to install. When that happens
// a 500 Internal Server Error is returned with the first error message.
if (!installedIntegrations.length) {
throw (settledResults[0] as PromiseRejectedResult).reason;
@@ -383,20 +383,11 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
? [plugins.cloud?.setup?.elasticsearchUrl]
: await getFallbackESUrl(services.esLegacyConfigService);
- if (request.headers.accept === 'application/x-tar') {
- return response.ok({
- headers: {
- 'content-type': 'application/x-tar',
- },
- body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }),
- });
- }
-
return response.ok({
headers: {
- 'content-type': 'application/yaml',
+ 'content-type': 'application/x-tar',
},
- body: generateAgentConfigYAML({ elasticsearchUrl, installedIntegrations }),
+ body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }),
});
},
});
@@ -423,7 +414,7 @@ async function ensureInstalledIntegrations(
if (installSource === 'registry') {
const installation = await packageClient.ensureInstalledPackage({ pkgName });
const pkg = installation.package;
- const inputs = await packageClient.getAgentPolicyInputs(pkg.name, pkg.version);
+ const config = await packageClient.getAgentPolicyConfigYAML(pkg.name, pkg.version);
const { packageInfo } = await packageClient.getPackage(pkg.name, pkg.version);
return {
@@ -431,7 +422,7 @@ async function ensureInstalledIntegrations(
pkgName: pkg.name,
pkgVersion: pkg.version,
title: packageInfo.title,
- inputs: inputs.filter((input) => input.type !== 'httpjson'),
+ config,
dataStreams:
packageInfo.data_streams?.map(({ type, dataset }) => ({ type, dataset })) ?? [],
kibanaAssets: pkg.installed_kibana,
@@ -447,19 +438,21 @@ async function ensureInstalledIntegrations(
pkgName,
pkgVersion: '1.0.0', // Custom integrations are always installed as version `1.0.0`
title: pkgName,
- inputs: [
- {
- id: `filestream-${pkgName}`,
- type: 'filestream',
- streams: [
- {
- id: `filestream-${pkgName}`,
- data_stream: dataStream,
- paths: integration.logFilePaths,
- },
- ],
- },
- ],
+ config: dump({
+ inputs: [
+ {
+ id: `filestream-${pkgName}`,
+ type: 'filestream',
+ streams: [
+ {
+ id: `filestream-${pkgName}`,
+ data_stream: dataStream,
+ paths: integration.logFilePaths,
+ },
+ ],
+ },
+ ],
+ }),
dataStreams: [dataStream],
kibanaAssets: [],
};
@@ -538,25 +531,6 @@ function parseIntegrationsTSV(tsv: string) {
);
}
-const generateAgentConfigYAML = ({
- elasticsearchUrl,
- installedIntegrations,
-}: {
- elasticsearchUrl: string[];
- installedIntegrations: InstalledIntegration[];
-}) => {
- return dump({
- outputs: {
- default: {
- type: 'elasticsearch',
- hosts: elasticsearchUrl,
- api_key: '${API_KEY}', // Placeholder to be replaced by bash script with the actual API key
- },
- },
- inputs: installedIntegrations.map(({ inputs }) => inputs).flat(),
- });
-};
-
const generateAgentConfigTar = ({
elasticsearchUrl,
installedIntegrations,
@@ -592,7 +566,7 @@ const generateAgentConfigTar = ({
path: `inputs.d/${integration.pkgName}.yml`,
mode: 0o644,
mtime: now,
- data: dump({ inputs: integration.inputs }),
+ data: integration.config,
})),
]);
};
diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts
index de2e7ce65fd2d..c9cded0805f65 100644
--- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts
+++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts
@@ -57,7 +57,7 @@ export const IntegrationRT = t.type({
pkgName: t.string,
pkgVersion: t.string,
title: t.string,
- inputs: t.array(t.unknown),
+ config: t.string,
dataStreams: t.array(
t.type({
type: t.string,