SNOW-920994 add configuration for tests on jenkins

.gitignore

@@ -295,7 +295,6 @@ __pycache__/
 *.xsd.cs
 
 # Unencrypted file
-Snowflake.Data.Tests/parameters.json
 *.xml
 
 # WhiteSource

Jenkinsfile

@@ -0,0 +1,67 @@
+import groovy.json.JsonOutput
+
+
+timestamps {
+  node('regular-memory-node') {
+    stage('checkout') {
+      scmInfo = checkout scm
+      println("${scmInfo}")
+      env.GIT_BRANCH = scmInfo.GIT_BRANCH
+      env.GIT_COMMIT = scmInfo.GIT_COMMIT
+    }
+
+    stage('Build') {
+      withCredentials([
+        usernamePassword(credentialsId: '063fc85b-62a6-4181-9d72-873b43488411', usernameVariable: 'AWS_ACCESS_KEY_ID', passwordVariable: 'AWS_SECRET_ACCESS_KEY'),
+        string(credentialsId: 'a791118f-a1ea-46cd-b876-56da1b9bc71c',variable: 'NEXUS_PASSWORD')
+        ]) {
+        sh '''\
+        |#!/bin/bash -e
+        |export GIT_BRANCH=${GIT_BRANCH}
+        |export GIT_COMMIT=${GIT_COMMIT}
+        |$WORKSPACE/ci/build.sh
+        '''.stripMargin()
+      }
+    }
+    params = [
+      string(name: 'svn_revision', value: 'main'),
+      string(name: 'branch', value: 'main'),
+      string(name: 'client_git_commit', value: scmInfo.GIT_COMMIT),
+      string(name: 'client_git_branch', value: scmInfo.GIT_BRANCH),
+      string(name: 'TARGET_DOCKER_TEST_IMAGE', value: 'dotnet-centos7-net6'),
+      string(name: 'parent_job', value: env.JOB_NAME),
+      string(name: 'parent_build_number', value: env.BUILD_NUMBER)
+    ]
+    stage('Test') {
+      build job: 'RT-LanguageDotnet-PC',parameters: params
+    }
+  }
+}
+
+
+pipeline {
+  agent { label 'regular-memory-node' }
+  options { timestamps() }
+  environment {
+    COMMIT_SHA_LONG = sh(returnStdout: true, script: "echo \$(git rev-parse " + "HEAD)").trim()
+
+    // environment variables for semgrep_agent (for findings / analytics page)
+    // remove .git at the end
+    // remove SCM URL + .git at the end
+
+    BASELINE_BRANCH = "${env.CHANGE_TARGET}"
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        checkout scm
+      }
+    }
+  }
+}
+
+def wgetUpdateGithub(String state, String folder, String targetUrl, String seconds) {
+    def ghURL = "https://api.github.com/repos/snowflakedb/snowflake-connector-net/statuses/$COMMIT_SHA_LONG"
+    def data = JsonOutput.toJson([state: "${state}", context: "jenkins/${folder}",target_url: "${targetUrl}"])
+    sh "wget ${ghURL} --spider -q --header='Authorization: token $GIT_PASSWORD' --post-data='${data}'"
+}

Snowflake.Data.Tests/Snowflake.Data.Tests.csproj

@@ -14,6 +14,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="JunitXml.TestLogger" Version="3.1.12" />
     <PackageReference Include="Moq" Version="4.18.4" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.7.1" />
     <PackageReference Include="NUnit" Version="3.13.3" />

Snowflake.Data.Tests/parameters.json

@@ -0,0 +1,14 @@
+{
+    "testconnection": {
+        "SNOWFLAKE_TEST_HOST": "snowflake.reg.local",
+        "SNOWFLAKE_TEST_PROTOCOL":  "http",
+        "SNOWFLAKE_TEST_PORT":      "8082",
+        "SNOWFLAKE_TEST_USER":      "snowman",
+        "SNOWFLAKE_TEST_PASSWORD":  "test",
+        "SNOWFLAKE_TEST_ACCOUNT":   "s3testaccount",
+        "SNOWFLAKE_TEST_WAREHOUSE": "regress",
+        "SNOWFLAKE_TEST_DATABASE":  "testdb",
+        "SNOWFLAKE_TEST_SCHEMA":    "testschema",
+        "SNOWFLAKE_TEST_ROLE":      "sysadmin"
+    }
+}

ci/_init.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -e
+
+export PLATFORM=$(echo $(uname) | tr '[:upper:]' '[:lower:]')
+export INTERNAL_REPO=nexus.int.snowflakecomputing.com:8086
+if [[ -z "$GITHUB_ACTIONS" ]]; then
+    # Use the internal Docker Registry
+    export DOCKER_REGISTRY_NAME=$INTERNAL_REPO/docker
+    export WORKSPACE=${WORKSPACE:-/tmp}
+else
+    # Use Docker Hub
+    export DOCKER_REGISTRY_NAME=snowflakedb
+    export WORKSPACE=$GITHUB_WORKSPACE
+fi
+
+export DRIVER_NAME=dotnet
+
+# Build images
+BUILD_IMAGE_VERSION=1
+
+# Test Images
+TEST_IMAGE_VERSION=1
+
+declare -A BUILD_IMAGE_NAMES=(
+    [$DRIVER_NAME-centos7-net6]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-net6-build:$BUILD_IMAGE_VERSION
+)
+export BUILD_IMAGE_NAMES
+
+declare -A TEST_IMAGE_NAMES=(
+    [$DRIVER_NAME-centos7-net6]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-net6-test:$TEST_IMAGE_VERSION
+)
+export TEST_IMAGE_NAMES

ci/build.sh

@@ -0,0 +1,40 @@
+#!/bin/bash -e
+#
+# Build Dotnet driver
+#
+set -o pipefail
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source $THIS_DIR/_init.sh
+source $THIS_DIR/scripts/login_internal_docker.sh
+
+if [[ -z "$GITHUB_ACTIONS" ]]; then
+    export GIT_URL=${GIT_URL:-https://github.com/snowflakedb/snowflake-connector-net.git}
+    export GIT_BRANCH=${GIT_BRANCH:-origin/$(git rev-parse --abbrev-ref HEAD)}
+    export GIT_COMMIT=${GIT_COMMIT:-$(git rev-parse HEAD)}
+else
+    export GIT_URL=https://github.com/${GITHUB_REPOSITORY}.git
+    export GIT_BRANCH=origin/$(basename ${GITHUB_REF})
+    export GIT_COMMIT=${GITHUB_SHA}
+fi
+
+echo "GIT_URL: $GIT_URL, GIT_BRANCH: $GIT_BRANCH, GIT_COMMIT: $GIT_COMMIT"
+
+for name in "${!BUILD_IMAGE_NAMES[@]}"; do
+    echo "[INFO] Building $DRIVER_NAME on $name"
+    docker pull "${BUILD_IMAGE_NAMES[$name]}"
+    docker run \
+        -v $(cd $THIS_DIR/.. && pwd):/mnt/host \
+        -v $WORKSPACE:/mnt/workspace \
+        -e LOCAL_USER_ID=$(id -u $USER) \
+        -e GIT_URL \
+        -e GIT_BRANCH \
+        -e GIT_COMMIT \
+        -e AWS_ACCESS_KEY_ID \
+        -e AWS_SECRET_ACCESS_KEY \
+        -e GITHUB_ACTIONS \
+        -e GITHUB_SHA \
+        -e GITHUB_REF \
+        -e GITHUB_EVENT_NAME \
+        "${BUILD_IMAGE_NAMES[$name]}" \
+        "/mnt/host/ci/container/build_component.sh"
+done

ci/container/build_component.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+#
+# Build Dotnet Driver
+#
+set -o pipefail
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SOURCE_ROOT=${SOURCE_ROOT:-/mnt/host}
+
+cd $SOURCE_ROOT
+echo "[INFO] Running dotnet restore"
+dotnet restore
+echo "[INFO] Building"
+dotnet build

ci/container/test_component.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -e
+
+set -o pipefail
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+export SOURCE_ROOT=${SOURCE_ROOT:-/mnt/host}
+export WORKSPACE=${WORKSPACE:-/mnt/workspace}
+
+cd $SOURCE_ROOT
+# todo unignore MaxLobSizeIT after SNOW-1058345 is fixed
+dotnet test -f net6.0 -l "console;verbosity=normal" --logger:"junit;LogFilePath=$WORKSPACE/junit-dotnet.xml" --filter "FullyQualifiedName!=Snowflake.Data.Tests.IntegrationTests.MaxLobSizeIT" 

ci/image/Dockerfile.dotnet-centos7-net6-build

@@ -0,0 +1,47 @@
+FROM centos:7
+
+USER root
+WORKDIR /
+
+# update OS and install basic tools
+RUN yum -y update && \
+    yum -y install epel-release && \
+    yum -y install centos-release-scl && \
+    yum -y install git && \
+    yum -y install which && \
+    yum -y install zstd && \
+    yum -y install jq \
+
+# python
+RUN yum -y install python36
+RUN python3 -V
+RUN pip3 install -U pip
+RUN pip3 install -U snowflake-connector-python
+
+# aws
+RUN pip3 install -U awscli
+RUN aws --version
+
+# gosu
+RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.11/gosu-amd64"
+RUN chmod +x /usr/local/bin/gosu
+
+# Install build tools
+RUN yum -y groupinstall 'Development Tools'
+RUN yum -y install centos-release-scl
+RUN yum -y install devtoolset-8-gcc*
+SHELL [ "/usr/bin/scl", "enable", "devtoolset-8"]
+
+# dotnet-sdk-6.0
+RUN rpm -Uvh https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm
+RUN yum -y install --exclude 1:openssl-libs-1.0.2k-26.el7_9.x86_64 dotnet-sdk-6.0
+
+# workspace
+RUN mkdir -p /home/user && \
+    chmod 777 /home/user
+WORKDIR /home/user
+
+# entry point
+COPY scripts/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

ci/image/Dockerfile.dotnet-centos7-net6-test

@@ -0,0 +1 @@
+FROM nexus.int.snowflakecomputing.com:8086/docker/client-dotnet-centos7-net6-build:1

ci/image/build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -e
+#
+# Build Docker images
+#
+set -o pipefail
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source $THIS_DIR/../_init.sh
+
+for name in "${!BUILD_IMAGE_NAMES[@]}"; do
+    docker build \
+        --file $THIS_DIR/Dockerfile.$name-build \
+        --label snowflake \
+        --label $DRIVER_NAME \
+        --tag ${BUILD_IMAGE_NAMES[$name]} .
+done
+
+for name in "${!TEST_IMAGE_NAMES[@]}"; do
+    docker build \
+        --file $THIS_DIR/Dockerfile.$name-test \
+        --label snowflake \
+        --label $DRIVER_NAME \
+        --tag ${TEST_IMAGE_NAMES[$name]} .
+done

ci/image/scripts/entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -x
+# Add local user
+# Either use the LOCAL_USER_ID if passed in at runtime or
+# fallback
+
+USER_ID=${LOCAL_USER_ID:-9001}
+
+echo "Starting with UID : $USER_ID"
+useradd --shell /bin/bash -u $USER_ID -o -c "" -m user
+export HOME=/home/user
+
+exec /usr/local/bin/gosu user "$@"

ci/image/update.sh

@@ -0,0 +1,34 @@
+#!/bin/bash -e
+#
+# Build Docker images
+#
+set -o pipefail
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source $THIS_DIR/../_init.sh
+
+source $THIS_DIR/../scripts/login_internal_docker.sh
+
+for image in $(docker images --format "{{.ID}},{{.Repository}}:{{.Tag}}" | grep "nexus.int.snowflakecomputing.com" | grep "client-$DRIVER_NAME"); do
+    target_id=$(echo $image | awk -F, '{print $1}')
+    target_name=$(echo $image | awk -F, '{print $2}')
+    for name in "${!BUILD_IMAGE_NAMES[@]}"; do
+        if [[ "$target_name" == "${BUILD_IMAGE_NAMES[$name]}" ]]; then
+            echo $name
+            docker_hub_image_name=$(echo ${BUILD_IMAGE_NAMES[$name]/$DOCKER_REGISTRY_NAME/snowflakedb})
+            set -x
+            docker tag $target_id $docker_hub_image_name
+            set +x
+            docker push "${BUILD_IMAGE_NAMES[$name]}"
+        fi
+    done
+    for name in "${!TEST_IMAGE_NAMES[@]}"; do
+        if [[ "$target_name" == "${TEST_IMAGE_NAMES[$name]}" ]]; then
+            echo $name
+            docker_hub_image_name=$(echo ${TEST_IMAGE_NAMES[$name]/$DOCKER_REGISTRY_NAME/snowflakedb})
+            set -x
+            docker tag $target_id $docker_hub_image_name
+            set +x
+            docker push "${TEST_IMAGE_NAMES[$name]}"
+        fi
+    done
+done

ci/scripts/login_internal_docker.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+#
+# Login the Internal Docker Registry
+#
+if [[ -z "$GITHUB_ACTIONS" ]]; then
+    echo "[INFO] Login the internal Docker Resistry"
+    NEXUS_USER=${USERNAME:-jenkins}
+    if [[ -z "$NEXUS_PASSWORD" ]]; then
+        echo "[ERROR] Set NEXUS_PASSWORD to your LDAP password to access the internal repository!"
+        exit 1
+    fi
+    if ! docker login --username "$NEXUS_USER" --password "$NEXUS_PASSWORD" $INTERNAL_REPO; then
+        echo "[ERROR] Failed to connect to the nexus server. Verify the environment variable NEXUS_PASSWORD is set correctly for NEXUS_USER: $NEXUS_USER"
+        exit 1
+    fi
+else
+    echo "[INFO] No login the internal Docker Registry"
+fi

ci/test.sh

@@ -0,0 +1,54 @@
+#!/bin/bash -e
+
+set -o pipefail
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ROOT_DIR="$(cd "${THIS_DIR}/.." && pwd)"
+export WORKSPACE=${WORKSPACE:-/tmp}
+
+source $THIS_DIR/_init.sh
+source $THIS_DIR/scripts/login_internal_docker.sh
+
+echo "Use /sbin/ip"
+IP_ADDR=$(/sbin/ip -4 addr show scope global dev eth0 | grep inet | awk '{print $2}' | cut -d / -f 1)
+
+declare -A TARGET_TEST_IMAGES
+if [[ -n "$TARGET_DOCKER_TEST_IMAGE" ]]; then
+    echo "[INFO] TARGET_DOCKER_TEST_IMAGE: $TARGET_DOCKER_TEST_IMAGE"
+    IMAGE_NAME=${TEST_IMAGE_NAMES[$TARGET_DOCKER_TEST_IMAGE]}
+    if [[ -z "$IMAGE_NAME" ]]; then
+        echo "[ERROR] The target platform $TARGET_DOCKER_TEST_IMAGE doesn't exist. Check $THIS_DIR/_init.sh"
+        exit 1
+    fi
+    TARGET_TEST_IMAGES=([$TARGET_DOCKER_TEST_IMAGE]=$IMAGE_NAME)
+else
+    echo "[ERROR] Set TARGET_DOCKER_TEST_IMAGE to the docker image name to run the test"
+    for name in "${!TEST_IMAGE_NAMES[@]}"; do
+        echo "  " $name
+    done
+    exit 2
+fi
+
+for name in "${!TARGET_TEST_IMAGES[@]}"; do
+    echo "[INFO] Testing $DRIVER_NAME on $name"
+    docker container run \
+        --rm \
+        --add-host=snowflake.reg.local:${IP_ADDR} \
+        --add-host=s3testaccount.reg.local:${IP_ADDR} \
+        -v $ROOT_DIR:/mnt/host \
+        -v $WORKSPACE:/mnt/workspace \
+        -e LOCAL_USER_ID=$(id -u ${USER}) \
+        -e GIT_COMMIT \
+        -e GIT_BRANCH \
+        -e GIT_URL \
+        -e AWS_ACCESS_KEY_ID \
+        -e AWS_SECRET_ACCESS_KEY \
+        -e GITHUB_ACTIONS \
+        -e GITHUB_SHA \
+        -e GITHUB_REF \
+        -e RUNNER_TRACKING_ID \
+        -e JOB_NAME \
+        -e BUILD_NUMBER \
+        ${TEST_IMAGE_NAMES[$name]} \
+        /mnt/host/ci/container/test_component.sh
+        echo "[INFO] Test Results: $WORKSPACE/junit-dotnet.xml"
+done