Merge branch 'master' into 8.x

docker_test/VERSION

@@ -0,0 +1,4 @@
+Version: 1.0.1
+Released: 23 August 2024
+
+# License and Changelog at https://github.com/untergeek/es-docker-test-scripts

docker_test/common.bash

@@ -3,16 +3,19 @@
 # Source the common.bash file from the same path as the script
 source $(dirname "$0")/ansi_clean.bash
 
+#MANUAL_PROJECT_NAME=project_name
 DOCKER_PORT=9200
 LOCAL_PORT=9200
 URL_HOST=127.0.0.1
 ESUSR=elastic
 ENVFILE=.env
 CURLFILE=.kurl
 REPODOCKER=/media
-LIMIT=30
+REPOJSON=createrepo.json
+REPONAME=testing
+LIMIT=30  # How many seconds to wait to obtain the credentials
 IMAGE=docker.elastic.co/elasticsearch/elasticsearch
-HEAP=1GB
+MEMORY=1GB  # The heap will be half of this
 
 
 #############################
@@ -58,7 +61,7 @@ get_espw () {
     fi
 
     # Print the spinner to stderr (so it shows up)
-    printf "\r${spin:$s:1} ${seconds}s elapsed..." >&2
+    printf "\r${spin:$s:1} ${seconds}s elapsed (typically 15s - 25s)..." >&2
 
     # wait 1/10th of a second before looping again
     sleep 0.1
@@ -125,7 +128,7 @@ change_espw () {
 xpack_fork () {
 
   echo
-  echo "Getting credentials from the ${NAME} Elasticsearch container..."
+  echo "Getting Elasticsearch credentials from container \"${NAME}\"..."
   echo
 
   # Get the password from the change_espw function. It sets ESPWD
@@ -138,16 +141,22 @@ xpack_fork () {
   fi
 
   # Put envvars in ${ENVCFG}
+  echo "export ESCLIENT_USERNAME=${ESUSR}" >> ${ENVCFG}
   echo "export TEST_USER=${ESUSR}" >> ${ENVCFG}
   # We escape the quotes so we can include them in case of special characters
+  echo "export ESCLIENT_PASSWORD=\"${ESPWD}\"" >> ${ENVCFG}
   echo "export TEST_PASS=\"${ESPWD}\"" >> ${ENVCFG}
 
+
+  # Get the CA certificate and copy it to the PROJECT_ROOT
+  docker cp -q ${NAME}:/usr/share/elasticsearch/config/certs/http_ca.crt ${PROJECT_ROOT}
+
   # Put the credentials into ${CURLCFG}
   echo "-u ${ESUSR}:${ESPWD}" >> ${CURLCFG}
-  echo "-k" >> ${CURLCFG}
+  echo "--cacert ${CACRT}" >> ${CURLCFG}
 
-  # Get the CA certificate and copy it to the TESTPATH
-  docker cp -q ${NAME}:/usr/share/elasticsearch/config/certs/http_ca.crt ${TESTPATH}
+  # Complete
+  echo "Credentials captured!"
 }
 
 # Save original execution path
@@ -156,18 +165,34 @@ EXECPATH=$(pwd)
 # Extract the path for the script
 SCRIPTPATH="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P)"
 
-# Set the $PROJECT_ROOT/tests/integration path
+# Ensure we are in the script path
 cd ${SCRIPTPATH}
 
+# Get the directory name
+SCRIPTPATH_NAME=$(pwd | awk -F\/ '{print $NF}')
+
 # Go up a level
 cd ../
 
 # Find out what the last part of this directory is called
 PROJECT_NAME=$(pwd | awk -F\/ '{print $NF}')
 
+# Manually override the project name, if specified
+if [ "x${MANUAL_PROJECT_NAME}" != "x" ]; then
+  PROJECT_NAME=${MANUAL_PROJECT_NAME}
+fi
+
 # We should be at the project root dir now
 PROJECT_ROOT=$(pwd)
 
+if [ "${SCRIPTPATH_NAME}" != "docker_test" ]; then
+  echo "$0 is not in parent directory 'docker_test'"
+  echo "This could cause issues as that is expected."
+  echo "PROJECT_ROOT is now set to ${SCRIPTPATH}"
+  echo "You may want to set MANUAL_PROJECT_NAME in common.bash"
+  PROJECT_ROOT=${SCRIPTPATH}
+fi
+
 # If we have a tests/integration path, then we'll use that
 if [ -d "tests/integration" ]; then
   TESTPATH=${PROJECT_ROOT}/tests/integration
@@ -176,13 +201,12 @@ else
   TESTPATH=${SCRIPTPATH}
 fi
 
+# Set the CACRT var
+CACRT=${PROJECT_ROOT}/http_ca.crt
+
 # Set the .env file
 ENVCFG=${PROJECT_ROOT}/${ENVFILE}
-
-# Add TESTPATH to ${ENVCFG}, creating it or overwriting it
-echo "export TEST_PATH=${TESTPATH}" > ${ENVCFG}
-echo "export CA_CRT=${TESTPATH}/http_ca.crt" > ${ENVCFG}
-
+rm -rf ${ENVCFG}
 
 # Set the curl config file and ensure we're not reusing an old one
 CURLCFG=${SCRIPTPATH}/${CURLFILE}

docker_test/create.sh

@@ -1,8 +1,11 @@
 #!/bin/bash
 
+
 # Source the common.bash file from the same path as the script
 source $(dirname "$0")/common.bash
 
+echo
+
 # Test to see if we were passed a VERSION
 if [ "x${1}" == "x" ]; then
   echo "Error! No Elasticsearch version provided."
@@ -26,9 +29,13 @@ mkdir -p ${REPOLOCAL}
 ### Run Container ###
 #####################
 
+docker network rm -f ${NAME}-net > /dev/null 2>&1
+docker network create ${NAME}-net > /dev/null 2>&1
+
 # Start the container
-echo -en "\rStarting ${NAME} container... "
-docker run -d -it --name ${NAME} -m ${HEAP} \
+echo "Starting container \"${NAME}\" from ${IMAGE}:${VERSION}"
+echo -en "Container ID: "
+docker run -q -d -it --name ${NAME} --network ${NAME}-net -m ${MEMORY} \
   -p ${LOCAL_PORT}:${DOCKER_PORT} \
   -v ${REPOLOCAL}:${REPODOCKER} \
   -e "discovery.type=single-node" \
@@ -38,6 +45,18 @@ docker run -d -it --name ${NAME} -m ${HEAP} \
   -e "path.repo=${REPODOCKER}" \
 ${IMAGE}:${VERSION}
 
+# Set the URL
+URL=https://${URL_HOST}:${LOCAL_PORT}
+
+# Add TESTPATH to ${ENVCFG}, creating it or overwriting it
+echo "export CA_CRT=${PROJECT_ROOT}/http_ca.crt" >> ${ENVCFG}
+echo "export TEST_PATH=${TESTPATH}" >> ${ENVCFG}
+echo "export TEST_ES_SERVER=${URL}" >> ${ENVCFG}
+echo "export TEST_ES_REPO=${REPONAME}" >> ${ENVCFG}
+
+# Write some ESCLIENT_ environment variables to the .env file  
+echo "export ESCLIENT_CA_CERTS=${CACRT}" >> ${ENVCFG}
+echo "export ESCLIENT_HOSTS=${URL}" >> ${ENVCFG}
 
 # Set up the curl config file, first line creates a new file, all others append
 echo "-o /dev/null" > ${CURLCFG}
@@ -55,12 +74,6 @@ if [ $? -eq 1 ]; then
   exit 1
 fi
 
-# Set the URL
-URL=https://${URL_HOST}:${LOCAL_PORT}
-
-# Write the TEST_ES_SERVER environment variable to the .env file
-echo "export TEST_ES_SERVER=${URL}" >> ${ENVCFG}
-
 # We expect a 200 HTTP rsponse
 EXPECTED=200
 
@@ -74,6 +87,7 @@ ACTUAL=0
 COUNTER=0
 
 # Loop until we get our 200 code
+echo
 while [ "${ACTUAL}" != "${EXPECTED}" ] && [ ${COUNTER} -lt ${LIMIT} ]; do
 
   # Get our actual response
@@ -103,15 +117,63 @@ if [ "${ACTUAL}" != "${EXPECTED}" ]; then
 
 fi
 
+# Initialize trial license
+echo
+response=$(curl -s \
+  --cacert ${CACRT} -u "${ESUSR}:${ESPWD}" \
+  -XPOST "${URL}/_license/start_trial?acknowledge=true")
+
+expected='{"acknowledged":true,"trial_was_started":true,"type":"trial"}'
+if [ "$response" != "$expected" ]; then
+  echo "ERROR! Unable to start trial license!"
+else
+  echo -n "Trial license started and acknowledged. "
+fi
+
+# Set up snapshot repository. The following will create a JSON file suitable for use with
+# curl -d @filename
+
+rm -f ${REPOJSON}  
+
+# Build a pretty JSON object defining the repository settings
+echo    '{'                    >> $REPOJSON
+echo    '  "type": "fs",'      >> $REPOJSON
+echo    '  "settings": {'      >> $REPOJSON
+echo -n '    "location": "'    >> $REPOJSON
+echo -n "${REPODOCKER}"        >> $REPOJSON
+echo    '"'                    >> $REPOJSON
+echo    '  }'                  >> $REPOJSON
+echo    '}'                    >> $REPOJSON
+
+# Create snapshot repository
+response=$(curl -s \
+  --cacert ${CACRT} -u "${ESUSR}:${ESPWD}" \
+  -H 'Content-Type: application/json' \
+  -XPOST "${URL}/_snapshot/${REPONAME}?verify=false" \
+  --json \@${REPOJSON})
+
+expected='{"acknowledged":true}'
+if [ "$response" != "$expected" ]; then
+  echo "ERROR! Unable to create snapshot repository"
+else
+  echo "Snapshot repository \"${REPONAME}\" created."
+fi
+
+
 ##################
 ### Wrap it up ###
 ##################
 
 echo
 echo "${NAME} container is up using image elasticsearch:${VERSION}"
-
+echo "Ready to test!"
 echo
-echo "Environment variables are in \$PROJECT_ROOT/.env"
 
-echo
-echo "Ready to test!"
+if [ "$EXECPATH" == "$PROJECT_ROOT" ]; then
+  echo "Environment variables are in .env"
+elif [ "$EXECPATH" == "$SCRIPTPATH" ]; then
+  echo "\$PWD is $SCRIPTPATH."
+  echo "Environment variables are in ../.env"
+else
+  echo "Environment variables are in ${PROJECT_ROOT}/.env"
+fi

docker_test/destroy.sh

@@ -3,19 +3,28 @@
 # Source the common.bash file from the same path as the script
 source $(dirname "$0")/common.bash
 
+echo
+
 # Stop and remove the docker container
-echo "Stopping container ${NAME}..."
-echo "$(docker stop ${NAME}) stopped."
+RUNNING=$(docker ps -f name=${NAME} | grep -v NAMES | awk '{print $NF}')
+EXISTS=$(docker ps -af name=${NAME} | grep -v NAMES | awk '{print $NF}')
+if [ "${RUNNING}" == "${NAME}" ]; then
+  echo "Stopping container ${NAME}..."
+  echo "$(docker stop ${NAME}) stopped."
+fi
+if [ "${EXISTS}" == "${NAME}" ]; then
+  echo "Removing container ${NAME}..."
+  echo "$(docker rm -f ${NAME}) deleted."
+fi
 
-echo "Removing container ${NAME}..."
-echo "$(docker rm ${NAME}) deleted."
+# Delete Docker network
+docker network rm -f ${NAME}-net > /dev/null 2>&1
 
 # Delete .env file and curl config file
 echo "Deleting remaining files and directories"
 rm -rf ${REPOLOCAL}
-rm -f ${SCRIPTPATH}/Dockerfile
 rm -f ${ENVCFG}
 rm -f ${CURLCFG}
-rm -f ${TESTPATH}/http_ca.crt
+rm -f ${PROJECT_ROOT}/http_ca.crt
 
 echo "Cleanup complete."

docs/Changelog.rst

@@ -3,6 +3,20 @@
 Changelog
 =========
 
+8.15.1 (23 August 2024)
+-----------------------
+
+**Changes**
+
+  * Added ``commands.py`` as both a cleaner location for the ``show_all_options``
+    function, as well as a place it could be imported and re-used.
+  * Updated ``docs/example.rst`` and ``docs/tutorial.rst`` to reflect these
+    location changes.
+  * Updated ``pytest.ini`` to automatically look for and use ``.env`` for
+    environment variables for testing.
+  * Using versioned ``docker_test`` scripts now from
+    https://github.com/untergeek/es-docker-test-scripts
+
 8.15.0 (13 August 2024)
 -----------------------
 

docs/example.rst

@@ -307,3 +307,15 @@ This file is part of the source code and is at ``./es_client/cli_example.py``.
 
 .. literalinclude:: ../src/es_client/cli_example.py
   :language: python
+
+.. _included_commands:
+
+Included Commands
+=================
+
+This module is referenced by ``./es_client/cli_example.py`` and includes the
+``show-all-options`` and ``test-connection`` functions/commands available when
+running from the CLI.
+
+.. literalinclude:: ../src/es_client/commands.py
+  :language: python