Skip to content

Commit

Permalink
Add arm build images
Browse files Browse the repository at this point in the history
  • Loading branch information
ruomeiy-splunk committed Apr 22, 2024
1 parent 0de4bff commit 37d09f8
Show file tree
Hide file tree
Showing 6 changed files with 173 additions and 29 deletions.
32 changes: 27 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,13 @@ endif

# Linux Splunk arguments
SPLUNK_LINUX_FILENAME ?= splunk-${SPLUNK_VERSION}-${SPLUNK_BUILD}-Linux-${SPLUNK_ARCH}.tgz
SPLUNK_ARM_FILENAME ?= splunk-${SPLUNK_VERSION}-${SPLUNK_BUILD}-Linux-${SPLUNK_ARCH}.tgz
SPLUNK_LINUX_BUILD_URL ?= https://download.splunk.com/products/${SPLUNK_PRODUCT}/releases/${SPLUNK_VERSION}/linux/${SPLUNK_LINUX_FILENAME}
SPLUNK_ARM_BUILD_URL ?= https://download.splunk.com/products/${SPLUNK_PRODUCT}/releases/${SPLUNK_VERSION}/linux/${SPLUNK_ARM_FILENAME}
UF_LINUX_FILENAME ?= splunkforwarder-${SPLUNK_VERSION}-${SPLUNK_BUILD}-Linux-${SPLUNK_ARCH}.tgz
UF_LINUX_BUILD_URL ?= https://download.splunk.com/products/universalforwarder/releases/${SPLUNK_VERSION}/linux/${UF_LINUX_FILENAME}
UF_ARM_FILENAME ?= splunkforwarder-${SPLUNK_VERSION}-${SPLUNK_BUILD}-Linux-${SPLUNK_ARCH}.tgz
UF_ARM_BUILD_URL ?= https://download.splunk.com/products/universalforwarder/releases/${SPLUNK_VERSION}/linux/${UF_ARM_FILENAME}
# Windows Splunk arguments
SPLUNK_WIN_FILENAME ?= splunk-${SPLUNK_VERSION}-${SPLUNK_BUILD}-x64-release.msi
SPLUNK_WIN_BUILD_URL ?= https://download.splunk.com/products/${SPLUNK_PRODUCT}/releases/${SPLUNK_VERSION}/windows/${SPLUNK_WIN_FILENAME}
Expand All @@ -32,8 +36,8 @@ SCANNER_DATE := `date +%Y-%m-%d`
SCANNER_DATE_YEST := `TZ=GMT+24 +%Y:%m:%d`
SCANNER_VERSION := v8
SCANNER_LOCALIP := $(shell ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | awk '{print $1}' | head -n 1)
SCANNER_IMAGES_TO_SCAN := splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-redhat-8 uf-debian-9 uf-debian-10 uf-centos-7 uf-redhat-8 splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-redhat-8 uf-py23-debian-9 uf-py23-debian-10 uf-py23-centos-7 uf-py23-redhat-8
CONTAINERS_TO_SAVE := splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-redhat-8 uf-debian-9 uf-debian-10 uf-centos-7 uf-redhat-8 splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-redhat-8 uf-py23-debian-9 uf-py23-debian-10 uf-py23-centos-7 uf-py23-redhat-8
SCANNER_IMAGES_TO_SCAN := splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-redhat-8 splunk-arm64-amazon-linux-2023 uf-arm64-amazon-linux-2023 uf-debian-9 uf-debian-10 uf-centos-7 uf-redhat-8 splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-redhat-8 uf-py23-debian-9 uf-py23-debian-10 uf-py23-centos-7 uf-py23-redhat-8
CONTAINERS_TO_SAVE := splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-redhat-8 uf-debian-9 splunk-arm64-amazon-linux-2023 uf-arm64-amazon-linux-2023 uf-debian-10 uf-centos-7 uf-redhat-8 splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-redhat-8 uf-py23-debian-9 uf-py23-debian-10 uf-py23-centos-7 uf-py23-redhat-8
ifeq ($(shell uname), Linux)
SCANNER_FILE = clair-scanner_linux_amd64
else ifeq ($(shell uname), Darwin)
Expand All @@ -57,7 +61,7 @@ ansible:
@cat splunk-ansible/version.txt

##### Base images #####
base: base-debian-9 base-debian-10 base-centos-7 base-centos-8 base-redhat-8 base-windows-2016
base: base-debian-9 base-debian-10 base-centos-7 base-centos-8 base-redhat-8 base-windows-2016 base-arm64-amazon-linux-2023

base-debian-10:
docker build ${DOCKER_BUILD_FLAGS} -t base-debian-10:${IMAGE_VERSION} ./base/debian-10
Expand All @@ -80,6 +84,9 @@ base-redhat-8-armv8:
base-windows-2016:
docker build ${DOCKER_BUILD_FLAGS} -t base-windows-2016:${IMAGE_VERSION} ./base/windows-2016

base-arm64-amazon-linux-2023:
docker build ${DOCKER_BUILD_FLAGS} --platform=linux/arm64/v8 --label version=${SPLUNK_VERSION} -t base-arm64-amazon-linux-2023:${IMAGE_VERSION} ./base/amazon-linux-2023

##### Minimal images #####
minimal: minimal-debian-9 minimal-debian-10 minimal-centos-7 minimal-centos-8 minimal-redhat-8

Expand Down Expand Up @@ -157,7 +164,7 @@ bare-redhat-8: base-redhat-8
--target bare -t bare-redhat-8:${IMAGE_VERSION} .

##### Splunk images #####
splunk: ansible splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-centos-8 splunk-redhat-8
splunk: ansible splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-centos-8 splunk-redhat-8 splunk-arm64-amazon-linux-2023

splunk-debian-9: base-debian-9 ansible
docker build ${DOCKER_BUILD_FLAGS} \
Expand Down Expand Up @@ -201,6 +208,14 @@ splunk-windows-2016: base-windows-2016 ansible
--build-arg SPLUNK_BUILD_URL=${SPLUNK_WIN_BUILD_URL} \
-t splunk-windows-2016:${IMAGE_VERSION} .

splunk-arm64-amazon-linux-2023: base-arm64-amazon-linux-2023 ansible
docker build ${DOCKER_BUILD_FLAGS} \
--platform=linux/arm64/v8 \
-f splunk/common-files/Dockerfile \
--build-arg SPLUNK_BASE_IMAGE=base-arm64-amazon-linux-2023 \
--build-arg SPLUNK_BUILD_URL=${SPLUNK_ARM_BUILD_URL} \
-t splunk-arm64-amazon-linux-2023:${IMAGE_VERSION} .

##### UF images #####
uf: ansible uf-debian-9 uf-debian-10 uf-centos-7 uf-centos-8 uf-redhat-8

Expand Down Expand Up @@ -268,6 +283,13 @@ uf-windows-2016: base-windows-2016 ansible
--build-arg SPLUNK_BUILD_URL=${UF_WIN_BUILD_URL} \
-t uf-windows-2016:${IMAGE_VERSION} .

uf-arm64-amazon-linux-2023: base-arm64-amazon-linux-2023 ansible
docker build ${DOCKER_BUILD_FLAGS} \
--platform=linux/arm64/v8 \
-f uf/common-files/Dockerfile \
--build-arg SPLUNK_BASE_IMAGE=base-arm64-amazon-linux-2023 \
--build-arg SPLUNK_BUILD_URL=${UF_ARM_BUILD_URL} \
-t uf-arm64-amazon-linux-2023:${IMAGE_VERSION} .

##### Python 3 support #####
splunk-py23: splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-centos-8 splunk-py23-redhat-8
Expand Down Expand Up @@ -350,7 +372,7 @@ run_large_tests: run_large_tests_centos7 run_large_tests_redhat8 run_large_tests

test_centos7: clean ansible splunk-centos-7 uf-centos-7 test_setup run_small_tests_centos7 run_large_tests_centos7

test_redhat8: clean ansible splunk-redhat-8 uf-redhat-8 test_setup run_small_tests_redhat8 run_large_tests_redhat8
test_redhat8: clean ansible splunk-redhat-8 splunk-arm64-amazon-linux-2023 uf-arm64-amazon-linux-2023 uf-redhat-8 test_setup run_small_tests_redhat8 run_large_tests_redhat8

test_debian9: clean ansible splunk-debian-9 uf-debian-9 test_setup run_small_tests_debian9 run_large_tests_debian9

Expand Down
31 changes: 31 additions & 0 deletions base/amazon-linux-2023/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# Copyright 2018-2024 Splunk
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

ARG BASE_IMAGE=amazonlinux:2023
FROM ${BASE_IMAGE} as package

LABEL name="splunk" \
maintainer="[email protected]" \
vendor="splunk" \
release="1" \
summary="Amazon-Linux-2023" \
description="Splunk Enterprise is a platform for operational intelligence. Our software lets you collect, analyze, and act upon the untapped value of big data that your technology infrastructure, security systems, and business applications generate. It gives you insights to drive operational performance and business results."


COPY install.sh /install.sh

RUN mkdir /licenses \
&& curl -o /licenses/apache-2.0.txt https://www.apache.org/licenses/LICENSE-2.0.txt

RUN /install.sh && rm -rf /install.sh
102 changes: 102 additions & 0 deletions base/amazon-linux-2023/install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
#!/bin/bash
# Copyright 2018-2024 Splunk
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

# Generate UTF-8 char map and locale
# Reinstalling local English def for now, removed in minimal image: https://bugzilla.redhat.com/show_bug.cgi?id=1665251
dnf -y --nodocs install glibc-langpack-en

# Currently there is no access to the UTF-8 char map. The following command is commented out until
# the base container can generate the locale.
# localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
# We get around the gen above by forcing the language install, and then pointing to it.
export LANG=en_US.utf8

# Install utility packages
dnf -y --nodocs install wget sudo shadow-utils procps tar make gcc \
openssl-devel bzip2-devel libffi-devel findutils \
libssh-devel libcurl-devel glib2-devel ncurses-devel \
diffutils bzip2
# Patch security updates
dnf -y --nodocs update gnutls kernel-headers libdnf librepo libnghttp2 nettle \
libpwquality libxml2 systemd-libs lz4-libs curl \
rpm rpm-libs sqlite-libs cyrus-sasl-lib vim expat \
openssl-libs xz-libs zlib libsolv file-libs pcre \
libarchive libgcrypt libksba libstdc++ json-c gnupg

# Reinstall tzdata (originally stripped from minimal image): https://bugzilla.redhat.com/show_bug.cgi?id=1903219
dnf -y --nodocs reinstall tzdata || dnf -y --nodocs update tzdata

# Build and install busybox direct from the multiarch since EPEL isn't available yet for redhat8
cd ~
wget https://busybox.net/downloads/busybox-1.36.1.tar.bz2
bzip2 -d busybox-1.36.1.tar.bz2
tar -xf busybox-1.36.1.tar
cd busybox-1.36.1
make defconfig
make
cp busybox /bin/busybox
cd ~
rm -rf busybox-1.36.1.tar busybox-1.36.1/

## Install Python and necessary packages
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
python3 get-pip.py
rm -f get-pip.py
ln -sf /usr/bin/python3 /usr/bin/python

# Install splunk-ansible dependencies
pip3 -q --no-cache-dir install --upgrade Mako avro lxml protobuf
pip3 install setuptools
pip3 install six
pip3 install wheel
pip3 install requests
pip3 install cryptography==3.3.2
pip3 install jmespath
pip3 install urllib3==1.26.5
pip3 install ansible
cd /

# Remove tests packaged in python libs
find /usr/lib/ -depth \( -type d -a -not -wholename '*/ansible/plugins/test' -a \( -name test -o -name tests -o -name idle_test \) \) -exec rm -rf '{}' \;
find /usr/lib/ -depth \( -type f -a -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) -exec rm -rf '{}' \;
find /usr/lib/ -depth \( -type f -a -name 'wininst-*.exe' \) -exec rm -rf '{}' \;
ldconfig

# Cleanup
dnf remove -y make gcc openssl-devel bzip2-devel findutils glib2-devel glibc-devel cpp binutils \
keyutils-libs-devel krb5-devel libcom_err-devel libffi-devel libcurl-devel \
libselinux-devel libsepol-devel libssh-devel libverto-devel libxcrypt-devel \
ncurses-devel pcre2-devel zlib-devel diffutils bzip2
dnf clean all

# Enable busybox symlinks
cd /bin
BBOX_LINKS=( clear find diff hostname killall netstat nslookup ping ping6 readline route syslogd tail traceroute vi )
for item in "${BBOX_LINKS[@]}"
do
ln -s busybox $item || true
done
chmod u+s /bin/ping
groupadd sudo

echo "
## Allows people in group sudo to run all commands
%sudo ALL=(ALL) ALL" >> /etc/sudoers

# Clean
dnf clean all
rm -rf /install.sh /anaconda-post.log /var/log/anaconda/*
5 changes: 1 addition & 4 deletions splunk/common-files/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,9 @@ FROM ${SPLUNK_BASE_IMAGE}:latest as package
ARG SPLUNK_BUILD_URL
COPY splunk/common-files/make-minimal-exclude.py /tmp
RUN python /tmp/make-minimal-exclude.py ${SPLUNK_BUILD_URL} > /tmp/splunk-minimal-exclude.list \
&& echo "Downloading Splunk and validating the checksum at: ${SPLUNK_BUILD_URL}" \
&& echo "Downloading Splunk: ${SPLUNK_BUILD_URL}" \
&& wget -qO /tmp/`basename ${SPLUNK_BUILD_URL}` ${SPLUNK_BUILD_URL} \
&& wget -qO /tmp/splunk.tgz.sha512 ${SPLUNK_BUILD_URL}.sha512 \
&& cd /tmp \
&& echo "$(cat /tmp/splunk.tgz.sha512)" | sha512sum --check --status \
&& rm /tmp/splunk.tgz.sha512 \
&& mkdir -p /minimal/splunk/var /extras/splunk/var \
&& tar -C /minimal/splunk --strip 1 --exclude-from=/tmp/splunk-minimal-exclude.list -zxf /tmp/`basename ${SPLUNK_BUILD_URL}` \
&& tar -C /extras/splunk --strip 1 --wildcards --files-from=/tmp/splunk-minimal-exclude.list -zxf /tmp/`basename ${SPLUNK_BUILD_URL}` \
Expand Down
27 changes: 11 additions & 16 deletions splunk/common-files/make-minimal-exclude.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,7 @@

EXCLUDE_V7 = """*-manifest
*/bin/installit.py
*/bin/jars/*
*/bin/jsmin*
*/bin/*mongo*
*/3rdparty/Copyright-for-mongo*
*/bin/node*
*/bin/pcregextest*
*/etc/*.lic*
Expand All @@ -19,7 +16,6 @@
*/etc/apps/sample_app*
*/etc/apps/appsbrowser*
*/etc/apps/alert_webhook*
*/etc/apps/splunk_archiver*
*/etc/apps/splunk_monitoring_console*
*/lib/node_modules*
*/share/splunk/app_templates*
Expand All @@ -30,24 +26,23 @@
*/share/splunk/pdf*
*mrsparkle*"""

version_string = re.match(".*splunk-([0-9]+)\.([0-9]+)\.[0-9]+\.?[0-9]?-[0-9a-z]+-Linux-[0-9a-z_-]+.tgz", sys.argv[1])
major_version = None
minor_version = None
m = re.match(".*splunk-([0-9]+)\.([0-9]+)\.[0-9]+\.?[0-9]?-[0-9a-z]+-Linux-([0-9a-z_-]+).tgz", sys.argv[1])

if version_string:
major_version = version_string.group(1)
minor_version = version_string.group(2)

if major_version:
if m and m.group(1):
print(EXCLUDE_V7)
if int(major_version) == 7:
if int(m.group(1)) == 7:
print("*/bin/parsetest*")
if int(minor_version) < 3:
if int(m.group(2)) < 3:
print("*/etc/apps/framework*")
print("*/etc/apps/gettingstarted*")
else:
print("*/etc/apps/splunk_metrics_workspace*")
elif 7 < int(major_version) < 9:
elif int(m.group(1)) > 7:
print("*/etc/apps/splunk_metrics_workspace*")
if int(minor_version) < 1:
if int(m.group(1)) == 8 and int(m.group(2)) < 1:
print("*/bin/parsetest*")
if m.group(3) != "aarch64":
print("*/bin/jars/*")
print("*/bin/*mongo*")
print("*/3rdparty/Copyright-for-mongo*")
print("*/etc/apps/splunk_archiver*")
5 changes: 1 addition & 4 deletions uf/common-files/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,9 @@ ARG SPLUNK_BASE_IMAGE=base-debian-10
FROM ${SPLUNK_BASE_IMAGE}:latest as package
ARG SPLUNK_BUILD_URL
ENV SPLUNK_HOME=/opt/splunkforwarder
RUN echo "Downloading Splunk and validating the checksum at: ${SPLUNK_BUILD_URL}" \
RUN echo "Downloading Splunk: ${SPLUNK_BUILD_URL}" \
&& wget -qO /tmp/`basename ${SPLUNK_BUILD_URL}` ${SPLUNK_BUILD_URL} \
&& wget -qO /tmp/splunk.tgz.sha512 ${SPLUNK_BUILD_URL}.sha512 \
&& cd /tmp \
&& echo "$(cat /tmp/splunk.tgz.sha512)" | sha512sum --check --status \
&& rm /tmp/splunk.tgz.sha512 \
&& tar -C /opt -zxf /tmp/`basename ${SPLUNK_BUILD_URL}` \
&& mv ${SPLUNK_HOME}/etc ${SPLUNK_HOME}-etc \
&& mkdir -p ${SPLUNK_HOME}/etc ${SPLUNK_HOME}/var
Expand Down

0 comments on commit 37d09f8

Please sign in to comment.