Skip to content

Commit

Permalink
Hopefully this one works
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChrisspyB
ChrisspyB committed Oct 24, 2024
1 parent 956c70f commit 1e5196a
Showing 1 changed file with 96 additions and 114 deletions.
210 changes: 96 additions & 114 deletions tests/tools/callback_vs_scan.sh.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ export METKIT_RAW_PARAM=1 # when *isnt* this needed...?
binary="@CMAKE_BINARY_DIR@"
prefix="@CMAKE_INSTALL_PREFIX@"
export PLUGINS_MANIFEST_PATH="$binary/share/plugins"
export DYLD_LIBRARY_PATH="$binary/lib:$binary/lib64:$prefix/lib:$prefix/lib64:$DYLD_LIBRARY_PATH"
export LD_LIBRARY_PATH="$binary/lib:$binary/lib64:$prefix/lib:$prefix/lib64:$LD_LIBRARY_PATH"
export DYLD_LIBRARY_PATH="$binary/lib:$binary/lib64:$DYLD_LIBRARY_PATH"
export LD_LIBRARY_PATH="$binary/lib:$binary/lib64:$LD_LIBRARY_PATH"

export LOAD_PLUGINS="int.ecmwf.gribjump-plugin"

Expand Down Expand Up @@ -72,125 +72,107 @@ cat $selected $unselected > $testdata

selectedrequests=${bindir}/requests
export FDB5_CONFIG_FILE=${bindir}/config_root1.yaml
$fdbread --extract ${selected} /dev/null > $selectedrequests

# -------------------------------------
# 1: Reference FDB

# DEBUGGING Make sure we fail the CI
export DEBUG=1
export ECKIT_DEBUG=1
export ECKIT_EXCEPTION_DUMPS_BACKTRACE=1
# run ls on each of the LD_LIBRARY_PATH directories
for dir in $(echo $LD_LIBRARY_PATH | tr ':' ' '); do
if [ -d $dir ]; then
ls -l $dir
fi
done

$fdbread --extract ${selected} /dev/null

exit 1

# <OLD STUFF>
# $fdbread --extract ${selected} /dev/null > $selectedrequests

# # -------------------------------------
# # 1: Reference FDB

# echo "Writing to FDB without gribjump plugin or scan"
# export FDB5_CONFIG_FILE=${bindir}/config_root1.yaml
# $fdbwrite ${testdata}

# # # -------------------------------------
# # # 2: FDB with scan post-process

# echo "Writing to FDB without gribjump plugin, then scanning"
# export FDB5_CONFIG_FILE=${bindir}/config_root2.yaml
# $fdbwrite ${testdata}
# $gjscan $selectedrequests

# # -------------------------------------
# # 3: FDB with gribjump plugin

# echo "Writing to FDB with gribjump plugin"
# export FDB5_CONFIG_FILE=${bindir}/config_root3.yaml
# export FDB_ENABLE_GRIBJUMP=1
# $fdbwrite ${testdata}
# unset FDB_ENABLE_GRIBJUMP
echo "Writing to FDB without gribjump plugin or scan"
export FDB5_CONFIG_FILE=${bindir}/config_root1.yaml
$fdbwrite ${testdata}

# # -------------------------------------
# # 4: FDB with gribjump-scan-files

# echo "Writing to FDB without gribjump plugin, then scanning with gribjump-scan-files"
# export FDB5_CONFIG_FILE=${bindir}/config_root4.yaml
# $fdbwrite ${testdata}
# # 2: FDB with scan post-process

echo "Writing to FDB without gribjump plugin, then scanning"
export FDB5_CONFIG_FILE=${bindir}/config_root2.yaml
$fdbwrite ${testdata}
$gjscan $selectedrequests

# -------------------------------------
# 3: FDB with gribjump plugin

echo "Writing to FDB with gribjump plugin"
export FDB5_CONFIG_FILE=${bindir}/config_root3.yaml
export FDB_ENABLE_GRIBJUMP=1
$fdbwrite ${testdata}
unset FDB_ENABLE_GRIBJUMP

# -------------------------------------
# 4: FDB with gribjump-scan-files

echo "Writing to FDB without gribjump plugin, then scanning with gribjump-scan-files"
export FDB5_CONFIG_FILE=${bindir}/config_root4.yaml
$fdbwrite ${testdata}

# # get a list of all .data files in the fdb xxxx database

filelist=$(find ${bindir}/root4 -path '*xxxx*' -name '*.data' | sort)
$gjscanfiles $filelist

# -------------------------------------
# Inspect the .gribjump files from fdbs and compare.

files2=$(find ${bindir}/root2 -name "*.gribjump" | sort)
files3=$(find ${bindir}/root3 -name "*.gribjump" | sort)
files4=$(find ${bindir}/root4 -name "*.gribjump" | sort)

# Make sure they are the same number
Nfiles=$(echo $files2 | wc -w)

# Make sure that number for this test data.
if [ $Nfiles -ne 5 ]; then
echo "Expected 5 .gribjump files, got $Nfiles"
exit 1
fi

if [ $Nfiles -ne $(echo $files3 | wc -w) ]; then
echo "Different number of .gribjump files in 2 and 3"
exit 1
fi

if [ $Nfiles -ne $(echo $files4 | wc -w) ]; then
echo "Different number of .gribjump files in 2 and 4"
exit 1
fi

# Iterate over the files, make sure the first file matches the first, etc.
# Note: if we ever store the full URI in the .gribjump file, this will need to be updated, as this will be different in 2 and 3.
# In which case, use e.g., gribjump-dump-info for comparing.
for i in $(seq 1 $Nfiles); do
file_2=$(echo $files2 | cut -d' ' -f$i)
file_3=$(echo $files3 | cut -d' ' -f$i)
file_4=$(echo $files4 | cut -d' ' -f$i)
diff $file_2 $file_3
diff $file_2 $file_4
done

# # # get a list of all .data files in the fdb xxxx database
# -------------------------------------
# Extract from all fdbs and compare the output
extractrequests=${bindir}/extractrequests
export FDB5_CONFIG_FILE=${bindir}/config_root1.yaml
$fdbread --extract ${supporteddata} /dev/null > $extractrequests

# filelist=$(find ${bindir}/root4 -path '*xxxx*' -name '*.data' | sort)
# $gjscanfiles $filelist
ranges=${bindir}/ranges
echo "0-10,20-30,40-50" > $ranges

# # -------------------------------------
# # Inspect the .gribjump files from fdbs and compare.

# files2=$(find ${bindir}/root2 -name "*.gribjump" | sort)
# files3=$(find ${bindir}/root3 -name "*.gribjump" | sort)
# files4=$(find ${bindir}/root4 -name "*.gribjump" | sort)

# # Make sure they are the same number
# Nfiles=$(echo $files2 | wc -w)

# # Make sure that number for this test data.
# if [ $Nfiles -ne 5 ]; then
# echo "Expected 5 .gribjump files, got $Nfiles"
# exit 1
# fi

# if [ $Nfiles -ne $(echo $files3 | wc -w) ]; then
# echo "Different number of .gribjump files in 2 and 3"
# exit 1
# fi

# if [ $Nfiles -ne $(echo $files4 | wc -w) ]; then
# echo "Different number of .gribjump files in 2 and 4"
# exit 1
# fi

# # Iterate over the files, make sure the first file matches the first, etc.
# # Note: if we ever store the full URI in the .gribjump file, this will need to be updated, as this will be different in 2 and 3.
# # In which case, use e.g., gribjump-dump-info for comparing.
# for i in $(seq 1 $Nfiles); do
# file_2=$(echo $files2 | cut -d' ' -f$i)
# file_3=$(echo $files3 | cut -d' ' -f$i)
# file_4=$(echo $files4 | cut -d' ' -f$i)
# diff $file_2 $file_3
# diff $file_2 $file_4
# done
for i in $(seq 1 4); do
export FDB5_CONFIG_FILE=${bindir}/config_root${i}.yaml
$gjextract $extractrequests $ranges > ${bindir}/tmp
cat ${bindir}/tmp
sed -n '/Extracted values:/,$p' ${bindir}/tmp > ${bindir}/extract${i}.clean
done
rm ${bindir}/tmp

# # -------------------------------------
# # Extract from all fdbs and compare the output
# extractrequests=${bindir}/extractrequests
# export FDB5_CONFIG_FILE=${bindir}/config_root1.yaml
# $fdbread --extract ${supporteddata} /dev/null > $extractrequests

# ranges=${bindir}/ranges
# echo "0-10,20-30,40-50" > $ranges

# for i in $(seq 1 4); do
# export FDB5_CONFIG_FILE=${bindir}/config_root${i}.yaml
# $gjextract $extractrequests $ranges > ${bindir}/tmp
# cat ${bindir}/tmp
# sed -n '/Extracted values:/,$p' ${bindir}/tmp > ${bindir}/extract${i}.clean
# done
# rm ${bindir}/tmp

# diff ${bindir}/extract1.clean ${bindir}/extract2.clean
# diff ${bindir}/extract1.clean ${bindir}/extract3.clean
# diff ${bindir}/extract1.clean ${bindir}/extract4.clean
diff ${bindir}/extract1.clean ${bindir}/extract2.clean
diff ${bindir}/extract1.clean ${bindir}/extract3.clean
diff ${bindir}/extract1.clean ${bindir}/extract4.clean

# # -------------------------------------
# # Done! Tidy up
# for i in $(seq 1 4); do
# rm ${bindir}/extract${i}.clean
# rm -rf ${bindir}/root${i}
# done
# -------------------------------------
# Done! Tidy up
for i in $(seq 1 4); do
rm ${bindir}/extract${i}.clean
rm -rf ${bindir}/root${i}
done

# echo "Test finished"
echo "Test finished"

0 comments on commit 1e5196a

Please sign in to comment.