diff --git a/.github/workflows/test-thor.yml b/.github/workflows/test-thor.yml new file mode 100644 index 00000000000..f098a0dc9a5 --- /dev/null +++ b/.github/workflows/test-thor.yml @@ -0,0 +1,247 @@ +name: Thor ML Test + +on: + workflow_call: + inputs: + os: + type: string + description: 'Operating System' + required: false + default: 'ubuntu-22.04' + asset-name: + type: string + description: 'Asset Name' + required: false + default: 'build-docker-package' + dependencies: + type: string + description: 'Dependencies' + required: false + default: 'bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline pkg-config libtool autotools-dev automake git cmake xmlstarlet' + push: +jobs: + + main: + runs-on: ubuntu-22.04 + steps: + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + + # - name: Download Package + # uses: actions/download-artifact@v3 + # with: + # name: ${{ inputs.asset-name }} + # path: ${{ inputs.asset-name }} + - name: Download artifact + id: download-artifact + uses: dawidd6/action-download-artifact@v3 + with: + repo: hpcc-systems/HPCC-Platform + name: docker-ubuntu-22_04 + workflow: build-vcpkg.yml + workflow_conclusion: success + check_artifacts: true + + - name: Install Dependencies + shell: "bash" + run: | + sudo apt-get update + sudo apt-get install -y \ + git \ + wget \ + net-tools \ + tzdata \ + unzip \ + xvfb \ + libxi6 \ + default-jdk \ + gdb \ + bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline pkg-config libtool autotools-dev automake git cmake xmlstarlet + + - name: Install Package + shell: "bash" + run: | + sudo apt-get install -y -f ./*.deb + sudo chown -R $USER:$USER /opt/HPCCSystems + + sudo xmlstarlet ed -L -u 'Environment/Software/ThorCluster/@slavesPerNode' -v 2 -u 'Environment/Software/ThorCluster/@channelsPerSlave' -v 1 /etc/HPCCSystems/environment.xml + + - name: Install ML Dependencies + shell: "bash" + run: | + sudo apt install -y libsvm-dev libsvm-tools + sudo pip install GLM GNN Kmeans tensorflow numpy keras + wget https://cdn.hpccsystems.com/releases/CE-Candidate-9.6.12/bin/clienttools/hpccsystems-clienttools-community_9.6.12-1jammy_amd64_withsymbols.deb + wget https://cdn.hpccsystems.com/releases/CE-Candidate-9.6.10/bin/plugins/hpccsystems-plugin-eclblas_9.6.10-1jammy_amd64_withsymbols.deb + sudo dpkg -i --force-all hpccsystems-plugin-eclblas_9.6.10-1jammy_amd64_withsymbols.deb + continue-on-error: true + + - name: Start HPCC-Platform + shell: "bash" + run: | + export LANG="en_US.UTF-8" + sudo update-locale + sudo /etc/init.d/hpcc-init start + + + - name: Get test from Github + shell: "bash" + run: | + set -x + BUNDLES_TO_TEST=( "ML_Core" "PBblas" "DBSCAN" "TextVectors" "PBblas" "GLM" "GNN" "LearningTrees" "KMeans" "SupportVectorMachines" "LinearRegression" "LogisticRegression" ) + BUNDLES_COUNT=${#BUNDLES_TO_TEST[@]} + for ((i=0; i<$BUNDLES_COUNT; i++)) + do + echo "reached here" + BUNDLE_NAME=${BUNDLES_TO_TEST[i]} + BUNDLE_REPO="https://github.com/hpcc-systems/${BUNDLES_TO_TEST[i]}.git" + INSTALL_CMD="ecl bundle install -v --update --force $ML_INSTALL_EXTRA ${BUNDLE_REPO}" + + tryCountMax=5 + tryCount=$tryCountMax + tryDelay=1m + + while true + do + echo "reached in while too" + cRes=$( ${INSTALL_CMD} 2>&1 ) + retCode=$? + if [[ $retCode -ne 0 ]] + then + tryCount=$(( $tryCount-1 )) + echo "123reached321" + + + if [[ $tryCount -ne 0 ]] + then + echo "123reached" + sleep ${tryDelay} + continue + else + break; + fi + else + BUNDLE_VERSION=$( echo "${cRes}" | egrep "^$BUNDLE_NAME" | awk '{ print $2 }' ) + break + fi + done + done + - name: Run Tests + id: run + shell: "bash" + working-directory: /home/runner/.HPCCSystems/bundles/_versions/ + run: | + ProcessLog() + { + BUNDLE=$1 + TARGET=$2 + logfilename=$( ls -clr /home/runner/HPCCSystems-regression/log/thor.*.log | head -1 | awk '{ print $9 }' ) + failed=$(cat ${logfilename} | sed -n "s/^[[:space:]]*Failure:[[:space:]]*\([0-9]*\)[[:space:]]*$/\1/p") + + if [[ "$failed" -ne 0 ]] + then + echo "Bundle : ${BUNDLE}" >> /home/runner/HPCCSystems-regression/log/Failed_test.summary + cat ${logfilename} >> /home/runner/HPCCSystems-regression/log/Failed_test.summary + echo "uploadArtifact=true" >> $GITHUB_OUTPUT + fi + # Rename result log file to name of the bundle + logname=$(basename $logfilename) + bundlelogfilename=${logname//thor/$BUNDLE} + printf "%s, %s\n" "$logname" "$bundlelogfilename" + mv -v $logfilename /home/runner/HPCCSystems-regression/ml-$bundlelogfilename + + } + + BUNDLES_TO_TEST=( "ML_Core" "PBblas" "DBSCAN" "TextVectors" "PBblas" "GLM" "GNN" "LearningTrees" "KMeans" "SupportVectorMachines" "LinearRegression" "LogisticRegression" ) + while read bundle + do + bundleRunPath=${bundle%/ecl} # remove '/ecl' from the end of the $bundle + bundlePath=${bundleRunPath%/OBTTests}; # remove '/OBTTests' from the end of the $bundleRunPath if exists + bundleName=${bundlePath%/test} # remove '/test' from the end of the $bundlePath if exists + bundleName=$(basename $bundleName ) # remove path from $bundleName + + if [[ "$bundle" =~ "LearningTrees" ]] + then + # add a warning supression parameter in the file + for file in "RegressionTestModified.ecl" "ClassificationTestModified.ecl" + do + if [[ $( egrep -c '#ONWARNING\(30004' $bundle/$file ) -eq 0 ]] + then + pushd $bundle + cp -fv $file $file-back + # Insert a comment and the "#ONWARNING" after the Copyright header + sed -i '/## \*\//a \\n// Patched by the bundleTest on '"$( date '+%Y.%m.%d %H:%M:%S')"' \n#ONWARNING(30004, ignore); // Do not report execute time skew warning' $file + popd + fi + done + fi + + if [[ ! "${BUNDLES_TO_TEST[*]}" =~ "$bundleName" ]] + then + continue + fi + + pushd $bundleRunPath + + /opt/HPCCSystems/testing/regress/ecl-test run -t thor --config /opt/HPCCSystems/testing/regress/ecl-test.json --timeout 3600 -fthorConnectTimeout=3600 --pq 1 --ef ClassicTestModified.ecl,SVCTest.ecl,ClassificationTestModified.ecl + + retCode=$( echo $? ) + + if [ ${retCode} -eq 0 ] + then + ProcessLog "$bundleName" "thor" + fi + + popd + + done< <(find . -iname 'ecl' -type d | sort ) + # - name: Generate ZAP files + # if: ${{ ! inputs.Generate-Zap == '' }} + # run: | + # IFS=' ' read -a ML_GENERATE_ZAP_FOR <<< ${{ inputs.Generate-Zap }} + # if [ ${#ML_GENERATE_ZAP_FOR[@]} -ne 0 ] + # then + # for test in ${ML_GENERATE_ZAP_FOR[*]} + # do + # wuid=$(ecl getwuid -n $test --limit 1) + # if [ -n $wuid ] + # then + # ecl zapgen $wuid --path /home/runner/HPCCSystems-regression/zap --inc-thor-slave-logs + # fi + # done + # fi + + - name: Check for Core files + run: | + NUM_OF_ML_CORES=( $(sudo find /var/lib/HPCCSystems/ -iname 'core*' -mtime -1 -type f -exec printf "%s\n" '{}' \; ) ) + + GDB_CMD='gdb --batch --quiet -ex "set interactive-mode off" -ex "echo \n Backtrace for all threads\n==========================" -ex "thread apply all bt" -ex "echo \n Registers:\n==========================\n" -ex "info reg" -ex "echo \n Disas:\n==========================\n" -ex "disas" -ex "quit" ' + + if [ ${#NUM_OF_ML_CORES[@]} -ne 0 ] + then + for core in ${NUM_OF_ML_CORES[@]} + do + base=$( dirname $core ) + lastSubdir=${base##*/} + comp=${lastSubdir##my} + + #sudo gdb --batch --quiet -ex "set interactive-mode off" -ex "thread apply all bt" -ex "quit" "/opt/HPCCSystems/bin/${comp}" $core | sudo tee "$core.trace" 2>&1 + sudo ${GDB_CMD} "/opt/HPCCSystems/bin/${comp}" $core | sudo tee "$core.trace" 2>&1 + cp "$core.trace" /home/runner/HPCCSystems-regression/log/ + done + + # - name: Get test stat + # if: ${{ inputs.Get-Stat }} + # run: | + # ./QueryStat2.py -p /home/runner/HPCCSystems-regression/log/ -d '' -a --timestamp --compileTimeDetails 1 --graphTimings --allGraphItems --addHeader + + - name: ml-thor-test-logs-artifact + if: ${{ failure() || cancelled() || steps.run.outputs.uploadArtifact }} + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.asset-name }}-thor-test-logs + path: | + /home/runner/HPCCSystems-regression/log/* + /home/runner/HPCCSystems-regression/zap/* + if-no-files-found: ignore