Skip to content

Commit

Permalink
[Observability Onboarding] Display next steps (elastic#197179)
Browse files Browse the repository at this point in the history
## Summary

Display next steps after the auto-detect script completes.

## Screenshot

<img width="905" alt="Screenshot 2024-10-22 at 09 30 51"
src="https://github.com/user-attachments/assets/85108ea2-510a-46f7-8b79-135be91b586f">
  • Loading branch information
thomheymann authored Oct 24, 2024
1 parent 42de8c8 commit 48959e7
Show file tree
Hide file tree
Showing 6 changed files with 67 additions and 83 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ const createClientMock = (): jest.Mocked<PackageClient> => ({
installCustomIntegration: jest.fn(),
fetchFindLatestPackage: jest.fn(),
readBundledPackage: jest.fn(),
getAgentPolicyInputs: jest.fn(),
getAgentPolicyConfigYAML: jest.fn(),
getPackage: jest.fn(),
getPackageFieldsMetadata: jest.fn(),
getPackages: jest.fn(),
Expand Down
13 changes: 5 additions & 8 deletions x-pack/plugins/fleet/server/services/epm/package_service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import type {
InstallablePackage,
Installation,
RegistryPackage,
TemplateAgentPolicyInput,
} from '../../types';

import type { FleetAuthzRouteConfig } from '../security/types';
Expand Down Expand Up @@ -116,12 +115,12 @@ export interface PackageClient {
prerelease?: false;
}): Promise<PackageList>;

getAgentPolicyInputs(
getAgentPolicyConfigYAML(
pkgName: string,
pkgVersion?: string,
prerelease?: false,
ignoreUnverified?: boolean
): Promise<TemplateAgentPolicyInput[]>;
): Promise<string>;

reinstallEsAssets(
packageInfo: InstallablePackage,
Expand Down Expand Up @@ -284,7 +283,7 @@ class PackageClientImpl implements PackageClient {
return generatePackageInfoFromArchiveBuffer(archiveBuffer, 'application/zip');
}

public async getAgentPolicyInputs(
public async getAgentPolicyConfigYAML(
pkgName: string,
pkgVersion?: string,
prerelease?: false,
Expand All @@ -298,16 +297,14 @@ class PackageClientImpl implements PackageClient {
pkgVersion = pkg.version;
}

const { inputs } = await getTemplateInputs(
return getTemplateInputs(
this.internalSoClient,
pkgName,
pkgVersion,
'json',
'yml',
prerelease,
ignoreUnverified
);

return inputs;
}

public async getPackage(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ elastic_agent_config_path="/opt/Elastic/Agent/elastic-agent.yml"
elastic_agent_tmp_config_path="/tmp/elastic-agent-config.tar"
integration_names=()
integration_titles=()
config_files_with_password=()

OS="$(uname)"
ARCH="$(uname -m)"
Expand Down Expand Up @@ -155,7 +156,7 @@ download_elastic_agent() {
curl -L -O "$download_url" --silent --fail

if [ "$?" -eq 0 ]; then
printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz"
printf "\e[32;1m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz"
update_step_progress "ea-download" "complete"
else
update_step_progress "ea-download" "danger" "Failed to download Elastic Agent, see script output for error."
Expand All @@ -167,7 +168,7 @@ extract_elastic_agent() {
tar -xzf "${elastic_agent_artifact_name}.tar.gz"

if [ "$?" -eq 0 ]; then
printf "\e[1;32m✓\e[0m %s\n" "Archive extracted"
printf "\e[32;1m✓\e[0m %s\n" "Archive extracted"
update_step_progress "ea-extract" "complete"
else
update_step_progress "ea-extract" "danger" "Failed to extract Elastic Agent, see script output for error."
Expand All @@ -179,7 +180,7 @@ install_elastic_agent() {
"./${elastic_agent_artifact_name}/elastic-agent" install -f -n >/dev/null

if [ "$?" -eq 0 ]; then
printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")"
printf "\e[32;1m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")"
update_step_progress "ea-install" "complete"
else
update_step_progress "ea-install" "danger" "Failed to install Elastic Agent, see script output for error."
Expand Down Expand Up @@ -224,7 +225,7 @@ ensure_elastic_agent_healthy() {

backup_elastic_agent_config() {
if [ -f "$elastic_agent_config_path" ]; then
echo -e "\nExisting config found at $elastic_agent_config_path"
printf "\n%s \e[36m%s\e[0m\n" "Existing config found at" "$elastic_agent_config_path"

printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Create backup and continue installation?" "[Y/n] (default: Yes): "
read confirmation_reply
Expand All @@ -241,7 +242,7 @@ backup_elastic_agent_config() {
fi

if [ "$?" -eq 0 ]; then
printf "\n\e[1;32m✓\e[0m %s\n" "Backup saved to $backup_path"
printf "\n\e[32;1m✓\e[0m %s \e[36m%s\e[0m\n" "Backup saved to" "$backup_path"
else
update_step_progress "ea-config" "warning" "Failed to backup existing configuration"
fail "Failed to backup existing config - Try manually creating a backup or delete your existing config before re-running this script"
Expand Down Expand Up @@ -278,7 +279,7 @@ install_integrations() {
--output "$elastic_agent_tmp_config_path"

if [ "$?" -eq 0 ]; then
printf "\n\e[1;32m✓\e[0m %s\n" "Integrations installed"
printf "\n\e[32;1m✓\e[0m %s\n" "Integrations installed"
else
update_step_progress "ea-config" "warning" "Failed to install integrations"
fail "Failed to install integrations"
Expand All @@ -297,10 +298,15 @@ apply_elastic_agent_config() {
# Replace placeholder with the Ingest API key
sed -i='' "s/\${API_KEY}/$decoded_ingest_api_key/" "$elastic_agent_config_path"
if [ "$?" -eq 0 ]; then
printf "\e[1;32m✓\e[0m %s\n" "Config written to:"
tar --list --file "$elastic_agent_tmp_config_path" | grep '\.yml$' | while read -r file; do
echo " - $(dirname "$elastic_agent_config_path")/$file"
done
printf "\e[32;1m✓\e[0m %s\n" "Config files written to:"
while IFS= read -r file; do
local path="$(dirname "$elastic_agent_config_path")/$file"
printf " \e[36m%s\e[0m\n" "$path"
grep '<PASSWORD>' "$path" >/dev/null
if [ "$?" -eq 0 ]; then
config_files_with_password+=("$path")
fi
done < <(tar --list --file "$elastic_agent_tmp_config_path" | grep '\.yml$')

update_step_progress "ea-config" "complete"
else
Expand Down Expand Up @@ -585,4 +591,11 @@ printf "\n\e[1m%s\e[0m\n" "Waiting for healthy status..."
wait_for_elastic_agent_status
ensure_elastic_agent_healthy

printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running. You can now go back to Kibana and check for incoming logs."
printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running!"

printf "\n\e[1m%s\e[0m\n" "Next steps:"
printf "\n• %s\n" "Go back to Kibana and check for incoming data"
for path in "${config_files_with_password[@]}"; do
printf "\n• %s:\n \e[36m%s\e[0m\n" "Collect $(known_integration_title "$(basename "${path%.yml}")") metrics by adding your username and password to" "$path"
done
printf "\n• %s:\n \e[36;4m%s\e[0m\n" "For information on other standalone integration setups, visit" "https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html"
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
[system]
title=System Logs And Metrics
title=System

[nginx]
title=Nginx Logs
title=Nginx
patterns=
/var/log/nginx/access.log*
/var/log/nginx/error.log*

[apache]
title=Apache Logs
title=Apache
patterns=
/var/log/apache2/access.log*
/var/log/apache2/other_vhosts_access.log*
Expand All @@ -17,48 +17,48 @@ patterns=
/var/log/httpd/error_log*

[docker]
title=Docker Container Logs
title=Docker
patterns=
/var/lib/docker/containers/*/*-json.log
/var/run/docker.sock

[mysql]
title=MySQL Logs
title=MySQL
patterns=
/var/log/mysql/*error.log*
/var/log/mysqld.log*
/var/log/mysql/*-slow.log*
/var/lib/mysql/*-slow.log*

[postgresql]
title=PostgreSQL Logs
title=PostgreSQL
patterns=
/var/log/postgresql/postgresql-*-*.log*
/*/postgresql-logs/*.log
/etc/postgresql/*/main/postgresql.conf
/var/log/postgresql/postgresql-*-*.csv*

[redis]
title=Redis Logs
title=Redis
patterns=
/var/log/redis/redis-server.log*
/etc/redis/redis.conf

[haproxy]
title=HAProxy Logs
title=HAProxy
patterns=
/var/log/haproxy.log
/etc/haproxy/haproxy.cfg

[rabbitmq]
title=RabbitMQ Logs
title=RabbitMQ
patterns=
/var/log/rabbitmq/rabbit@*.log
/etc/rabbitmq/rabbitmq.conf
/etc/rabbitmq/rabbitmq.config

[kafka]
title=Kafka Logs
title=Kafka
patterns=
/var/log/kafka/server.log
/etc/kafka/server.properties
Expand All @@ -68,19 +68,19 @@ patterns=
/*/logs/kafka-*.log*

[mongodb]
title=MongoDB Logs
title=MongoDB
patterns=
/var/log/mongodb/mongod.log

[apache_tomcat]
title=Apache Tomcat Logs
title=Apache Tomcat
patterns=
/opt/tomcat/logs/localhost_access_log.*.txt
/opt/tomcat/logs/catalina.*.log
/opt/tomcat/logs/localhost.*.log

[prometheus]
title=Prometheus Server overview
title=Prometheus
patterns=
/var/log/prometheus/prometheus.log
/etc/prometheus/prometheus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -263,8 +263,8 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
*
* The request format is TSV (tab-separated values) to simplify parsing in bash.
*
* The response format is either a YAML file or a tar archive containing the Elastic Agent
* configuration, depending on the `Accept` header.
* The response format is a tar archive containing the Elastic Agent configuration, depending on the
* `Accept` header.
*
* Errors during installation are ignore unless all integrations fail to install. When that happens
* a 500 Internal Server Error is returned with the first error message.
Expand Down Expand Up @@ -348,7 +348,7 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
}
return acc;
}, []);
// Errors during installation are ignore unless all integrations fail to install. When that happens
// Errors during installation are ignored unless all integrations fail to install. When that happens
// a 500 Internal Server Error is returned with the first error message.
if (!installedIntegrations.length) {
throw (settledResults[0] as PromiseRejectedResult).reason;
Expand Down Expand Up @@ -383,20 +383,11 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
? [plugins.cloud?.setup?.elasticsearchUrl]
: await getFallbackESUrl(services.esLegacyConfigService);

if (request.headers.accept === 'application/x-tar') {
return response.ok({
headers: {
'content-type': 'application/x-tar',
},
body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }),
});
}

return response.ok({
headers: {
'content-type': 'application/yaml',
'content-type': 'application/x-tar',
},
body: generateAgentConfigYAML({ elasticsearchUrl, installedIntegrations }),
body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }),
});
},
});
Expand All @@ -423,15 +414,15 @@ async function ensureInstalledIntegrations(
if (installSource === 'registry') {
const installation = await packageClient.ensureInstalledPackage({ pkgName });
const pkg = installation.package;
const inputs = await packageClient.getAgentPolicyInputs(pkg.name, pkg.version);
const config = await packageClient.getAgentPolicyConfigYAML(pkg.name, pkg.version);
const { packageInfo } = await packageClient.getPackage(pkg.name, pkg.version);

return {
installSource,
pkgName: pkg.name,
pkgVersion: pkg.version,
title: packageInfo.title,
inputs: inputs.filter((input) => input.type !== 'httpjson'),
config,
dataStreams:
packageInfo.data_streams?.map(({ type, dataset }) => ({ type, dataset })) ?? [],
kibanaAssets: pkg.installed_kibana,
Expand All @@ -447,19 +438,21 @@ async function ensureInstalledIntegrations(
pkgName,
pkgVersion: '1.0.0', // Custom integrations are always installed as version `1.0.0`
title: pkgName,
inputs: [
{
id: `filestream-${pkgName}`,
type: 'filestream',
streams: [
{
id: `filestream-${pkgName}`,
data_stream: dataStream,
paths: integration.logFilePaths,
},
],
},
],
config: dump({
inputs: [
{
id: `filestream-${pkgName}`,
type: 'filestream',
streams: [
{
id: `filestream-${pkgName}`,
data_stream: dataStream,
paths: integration.logFilePaths,
},
],
},
],
}),
dataStreams: [dataStream],
kibanaAssets: [],
};
Expand Down Expand Up @@ -538,25 +531,6 @@ function parseIntegrationsTSV(tsv: string) {
);
}

const generateAgentConfigYAML = ({
elasticsearchUrl,
installedIntegrations,
}: {
elasticsearchUrl: string[];
installedIntegrations: InstalledIntegration[];
}) => {
return dump({
outputs: {
default: {
type: 'elasticsearch',
hosts: elasticsearchUrl,
api_key: '${API_KEY}', // Placeholder to be replaced by bash script with the actual API key
},
},
inputs: installedIntegrations.map(({ inputs }) => inputs).flat(),
});
};

const generateAgentConfigTar = ({
elasticsearchUrl,
installedIntegrations,
Expand Down Expand Up @@ -592,7 +566,7 @@ const generateAgentConfigTar = ({
path: `inputs.d/${integration.pkgName}.yml`,
mode: 0o644,
mtime: now,
data: dump({ inputs: integration.inputs }),
data: integration.config,
})),
]);
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ export const IntegrationRT = t.type({
pkgName: t.string,
pkgVersion: t.string,
title: t.string,
inputs: t.array(t.unknown),
config: t.string,
dataStreams: t.array(
t.type({
type: t.string,
Expand Down

0 comments on commit 48959e7

Please sign in to comment.