workaround: add flash_attn test to ci

tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml

@@ -255,6 +255,33 @@ jobs:
       arguments: -n $(Agent.Name) -d $HIP_VISIBLE_DEVICES -r $DRIVER_RENDER
     displayName: 'Check ROCm Environment'
 
+  # TODO: move to use ci_build/build.py driven tests
+  - task: CmdLine@2
+    inputs:
+      script: |-
+        docker run --rm \
+          --security-opt seccomp=unconfined \
+          --shm-size=1024m \
+          --device=/dev/kfd \
+          --device=/dev/dri/renderD$DRIVER_RENDER \
+          --group-add $(video) \
+          --group-add $(render) \
+          --user onnxruntimedev \
+          --volume $(Build.SourcesDirectory):/onnxruntime_src \
+          --volume $(Build.BinariesDirectory):/build \
+          -e OPENBLAS_NUM_THREADS=1 \
+          -e OPENMP_NUM_THREADS=1 \
+          -e MKL_NUM_THREADS=1 \
+          -e PYTHONPATH=/build/$(BuildConfig) \
+          onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-test \
+            /bin/bash -c "
+              set -ex; \
+              pip install /onnxruntime_src/tools/ci_build/requirements-transformers-test.txt; \
+              pytest /onnxruntime_src/onnxruntime/test/python/transformers/test_flash_attn_rocm.py -v -n 4 --reruns 1"
+      workingDirectory: $(Build.SourcesDirectory)
+    displayName: 'Run tranformers tests'
+    condition: succeededOrFailed()
+
   - task: CmdLine@2
     inputs:
       script: |-