Skip to content

Commit

Permalink
tools script additions and improvements (#104)
Browse files Browse the repository at this point in the history
Signed-off-by: Anthony Floeder <[email protected]>
  • Loading branch information
ajfloeder authored Aug 14, 2024
1 parent 18a1ca6 commit 32fe4da
Show file tree
Hide file tree
Showing 5 changed files with 250 additions and 24 deletions.
38 changes: 38 additions & 0 deletions tools/collectNVLogs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#!/bin/bash

# Copyright 2024 Hewlett Packard Enterprise Development LP
# Other additional copyright holders may be indicated within.
#
# The entirety of this work is licensed under the Apache License,
# Version 2.0 (the "License"); you may not use this file except
# in compliance with the License.
#
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# set -e
# set -o xtrace

shopt -s expand_aliases

logSources=("FLASH" "MEMLOG" "REGS" "THRD_STACK" "SYS_STACK" "THRDS" "NVHDR" "RAM")
switches=("/dev/switchtec0" "/dev/switchtec1")

for switch in "${switches[@]}"; do
PAX_ID=$(switchtec fabric gfms-dump "$switch" | grep "^PAX ID:" | awk '{print $3}')
if ! (( PAX_ID >= 0 && PAX_ID <= 1 )); then
echo "$PAX_ID not in range 0-1"
exit 1
fi

for logSource in "${logSources[@]}"; do
echo pax"$PAX_ID"-"$logSource".log
switchtec log-dump "$switch" pax"$PAX_ID"-"$logSource".log --type="$logSource"
done
done
42 changes: 41 additions & 1 deletion tools/rabbit-s.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

# Copyright 2020-2023 Hewlett Packard Enterprise Development LP
# Copyright 2020-2024 Hewlett Packard Enterprise Development LP
# Other additional copyright holders may be indicated within.
#
# The entirety of this work is licensed under the Apache License,
Expand Down Expand Up @@ -181,6 +181,46 @@ EOF
EOF
done
;;
hpecray-29)
for SESSION in "${SESSIONS[@]}"
do

# For this run, we want to enable some logging settings that we have been leaving off for a while.
# FABIOV - 0x82
# log -m 0x82 -s 3
# log -m 0x82 -s 3 -p on
# PTD - 0x53
# log -m 0x53 -s 3
# log -m 0x53 -s 3 -p on
# fabdbg -s pax
# fabdbg -s fio
# fabdbg -s gfms

$SSHPASS ssh root@$SYSTEM <<-EOF
screen -S $SESSION -X stuff "fabdbg -s pax\nfabdbg -s fio\nfabdbg -s gfms\nlog -m 0x82 -s 3\nlog -m 0x82 -s 3 -p on\nlog -m 0x53 -s 3\nlog -m 0x53 -s 3 -p on\n"
EOF
done
;;
hpecray-32)
for SESSION in "${SESSIONS[@]}"
do
# Enables medium severity and turns on logging for the PSC module
# log -m 0x54 -s 3
# log -m 0x54 -s 3 -p on

# Enables logs for the fabric debug modules
# fabdbg -s pax
# fabdbg -s fio
# fabdbg -s gfms

# Turn on the logging for all modules
# log -p on
$SSHPASS ssh root@$SYSTEM <<-EOF
screen -S $SESSION -X stuff "fabdbg -s pax\nfabdbg -s fio\nfabdbg -s gfms\nlog -m 0x54 -s 3\nlog -m 0x54 -s 3 -p on\nlog -p on\n"
EOF
done
;;

quit-sessions)
for SESSION in "${SESSIONS[@]}"
do
Expand Down
109 changes: 109 additions & 0 deletions tools/show-drive-ports.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
#!/bin/bash

# Copyright 2024 Hewlett Packard Enterprise Development LP
# Other additional copyright holders may be indicated within.
#
# The entirety of this work is licensed under the Apache License,
# Version 2.0 (the "License"); you may not use this file except
# in compliance with the License.
#
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

usage() {
cat <<EOF
Examine "lspci" status and report failing drive links
Usage: $0 [-h] [-v]
Arguments:
-v verbose output
-h display this help
Examples:
$0 -v
EOF
}

VERBOSE="false"
while getopts ":vh" OPTION
do
case "${OPTION}" in
'v')
VERBOSE="true"
;;
'h')
usage
exit 0
;;
*)
usage
exit 1
;;
esac
done
shift $((OPTIND - 1))

drive_links="04 82"

error_present="false"
for pax in $drive_links;
do
for drive in {0..8};
do
if [ $pax -eq 4 ] && [ $drive -eq 7 ]; then
continue
fi
if [ $pax -eq 82 ] && [ $drive -eq 4 ]; then
continue
fi

pci_result=$(lspci -s "$pax":"$drive".0 -vv | grep LnkSta:)
if [ $? -ne 0 ]; then
if [ $VERBOSE == "true" ]; then
printf "%d:%d.0 Error accessing drive\n" "$pax" "$drive";
fi
error_present="true"
continue
fi

access_error_result=$(echo "$pci_result" | grep "!!! Unknown header type")
if [ $? -eq 0 ]; then
if [ $VERBOSE == "true" ]; then
printf "%d:%d.0 Unable to retrieve link info\n" "$pax" "$drive"
fi
error_present="true"
continue
fi

width_zero_error_result=$(echo "$pci_result" | grep "Width x0")
if [ $? -eq 0 ]; then
if [ $VERBOSE == "true" ]; then
printf "%d:%d.0 Link down\n" "$pax" "$drive"
fi
error_present="true"
continue
fi

link_speed_error_result=$(echo "$pci_result" | grep 16GT)
if [ $? -ne 0 ]; then
if [ $VERBOSE == "true" ]; then
printf "%d:%d.0 Link speed too slow\n" "$pax" "$drive"
fi
error_present="true"
continue
fi
done;
done

if [ "$error_present" == "false" ]; then
printf "No errors\n"
else
printf "ERRORS FOUND\n"
fi
83 changes: 61 additions & 22 deletions tools/switch.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

# Copyright 2022-2023 Hewlett Packard Enterprise Development LP
# Copyright 2022-2024 Hewlett Packard Enterprise Development LP
# Other additional copyright holders may be indicated within.
#
# The entirety of this work is licensed under the Apache License,
Expand Down Expand Up @@ -28,7 +28,7 @@ usage() {
cat <<EOF
Run various switch command over all switches.
Usage: $0 [-h] [-t] COMMAND [ARGS...]
Usage: $0 [-hv] [-t] COMMAND [ARGS...]
Commands:
slot-info display slot status for each physical port
Expand All @@ -44,6 +44,7 @@ Commands:
Arguments:
-h display this help
-v verbose mode
-t time each command
Examples:
Expand Down Expand Up @@ -88,6 +89,20 @@ execute() {
done
}

getChassis() {
if [ "$VERBOSE" != "true" ]; then
CHASSIS=" "
return
fi

COMMAND=xhost-query.py
if command -v $COMMAND &> /dev/null; then
CHASSIS=$("$COMMAND" $(hostname) | cut -c -7)
else
CHASSIS="x****c*"
fi
}

getPAXID() {
local SWITCH_NAME=$1

Expand All @@ -104,6 +119,18 @@ getPAXID() {
fi
}

getPAXTemperature() {
local SWITCH_NAME=$1

# Make sure we can get the PAX ID
if [ ! "$(switchtec temp "$SWITCH_NAME")" ]; then
echo "Unable to retrieve PAX Temperature"
exit $?
fi

PAX_TEMPERATURE=$(switchtec temp "$SWITCH_NAME")
}

setDeviceName() {
DRIVES=()
getDriveList
Expand All @@ -118,6 +145,8 @@ setDeviceName() {
displayDriveSlotStatus() {
local SWITCH_NAME=$1

getChassis

# Physical slot ids are set into the hardware. These are the mappings
declare -a PAX0_DriveSlotFromPhysicalPort=(
# Drives
Expand Down Expand Up @@ -203,6 +232,8 @@ displayDriveSlotStatus() {
displayStatus() {
local SWITCH_NAME=$1

getChassis

# Physical slot ids are set into the hardware. These are the mappings
declare -a PAX0_ConnectedEPToPhysicalPort=(
# Drives
Expand All @@ -218,15 +249,15 @@ displayStatus() {

# Other Links
[0]="Interswitch Link "
[24]="Rabbit, x9000c?j7b0"
[32]="Compute 0, x9000c?s0b0n0"
[34]="Compute 1, x9000c?s0b1n0"
[36]="Compute 2, x9000c?s1b0n0"
[38]="Compute 3, x9000c?s1b1n0"
[40]="Compute 4, x9000c?s2b0n0"
[42]="Compute 5, x9000c?s2b1n0"
[44]="Compute 6, x9000c?s3b0n0"
[46]="Compute 7, x9000c?s3b1n0"
[24]="Rabbit, ${CHASSIS}r7b0n0"
[32]="Compute 0, ${CHASSIS}s0b0n0"
[34]="Compute 1, ${CHASSIS}s0b1n0"
[36]="Compute 2, ${CHASSIS}s1b0n0"
[38]="Compute 3, ${CHASSIS}s1b1n0"
[40]="Compute 4, ${CHASSIS}s2b0n0"
[42]="Compute 5, ${CHASSIS}s2b1n0"
[44]="Compute 6, ${CHASSIS}s3b0n0"
[46]="Compute 7, ${CHASSIS}s3b1n0"
)
declare -a PAX1_ConnectedEPToPhysicalPort=(
# Drives
Expand All @@ -242,23 +273,28 @@ displayStatus() {

# Other Links
[0]="Interswitch Link "
[24]="Rabbit, x9000c?j7b0"
[32]="Compute 8, x9000c?s4b0n0"
[34]="Compute 9, x9000c?s4b1n0"
[36]="Compute 10, x9000c?s5b0n0"
[38]="Compute 11, x9000c?s5b1n0"
[40]="Compute 12, x9000c?s6b0n0"
[42]="Compute 13, x9000c?s6b1n0"
[44]="Compute 14, x9000c?s7b0n0"
[46]="Compute 15, x9000c?s7b1n0"
[24]="Rabbit, ${CHASSIS}r7b0n0"
[32]="Compute 8, ${CHASSIS}s4b0n0"
[34]="Compute 9, ${CHASSIS}s4b1n0"
[36]="Compute 10, ${CHASSIS}s5b0n0"
[38]="Compute 11, ${CHASSIS}s5b1n0"
[40]="Compute 12, ${CHASSIS}s6b0n0"
[42]="Compute 13, ${CHASSIS}s6b1n0"
[44]="Compute 14, ${CHASSIS}s7b0n0"
[46]="Compute 15, ${CHASSIS}s7b1n0"
)

getPAXID "$SWITCH_NAME"

mapfile -t physicalPortIdStrings < <(switchtec status "$SWITCH_NAME" | grep "Phys Port ID:")

local physicalPortString
printf "DEVICE: %s PAX_ID: %d\n\n" "$SWITCH_NAME" "$PAX_ID"
if [ "$VERBOSE" == "true" ]; then
getPAXTemperature "$SWITCH_NAME"
printf "DEVICE: %s PAX_ID: %d TEMP: %s\n\n" "$SWITCH_NAME" "$PAX_ID" "$PAX_TEMPERATURE"
else
printf "DEVICE: %s PAX_ID: %d\n\n" "$SWITCH_NAME" "$PAX_ID"
fi
printf "Switch Connection \tStatus\n"
printf "===========================\t======\n"
for physicalPortString in "${physicalPortIdStrings[@]}";
Expand All @@ -285,13 +321,16 @@ displayStatus() {


alias TIME=""
while getopts "th:" OPTION
while getopts "tvh:" OPTION
do
case "${OPTION}" in
't')
alias TIME=time
export TIMEFORMAT='%3lR'
;;
'v')
export VERBOSE="true"
;;
'h',*)
usage
exit 0
Expand Down
2 changes: 1 addition & 1 deletion tools/timestamp-log.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

# Copyright 2024 Hewlett Packard Enterprise Development LP
# Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
# Other additional copyright holders may be indicated within.
#
# The entirety of this work is licensed under the Apache License,
Expand Down

0 comments on commit 32fe4da

Please sign in to comment.