Skip to content

Commit

Permalink
Fix apache#504 - Build images before running e2e (apache#546)
Browse files Browse the repository at this point in the history
  • Loading branch information
ricardozanini authored and rgdoliveira committed Oct 24, 2024
1 parent c728a6c commit bab8c7c
Show file tree
Hide file tree
Showing 95 changed files with 707 additions and 239 deletions.
2 changes: 1 addition & 1 deletion .ci/jenkins/Jenkinsfile.e2e.cluster
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ pipeline {
helper.checkoutRepo()

assert getTestImage(): 'Please provide a Test image'
container.pullImage(getTestImage()) // Verify image exists
container.pullImage(getTestImage()) // Verify image namespaceExists
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,8 @@ jobs:
make deploy IMG=${{ env.OPERATOR_IMAGE_NAME }}
kubectl wait pod -A -l control-plane=sonataflow-operator --for condition=Ready
- name: Run E2E Tests for Non-Persistent Flows
run: make test-e2e label=flows-non-persistence
- name: Run E2E Tests for Ephemeral Flows
run: make test-e2e label=flows-ephemeral

- name: Run E2E Tests for Persistent Flows
run: make test-e2e label=flows-persistence
Expand Down
19 changes: 11 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,9 @@ vet: ## Run go vet against code.

.PHONY: test
test: manifests generate envtest test-api ## Run tests.
@$(MAKE) addheaders
@$(MAKE) vet
@$(MAKE) fmt
@echo "🔍 Running controller tests..."
@KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \
go test $(shell go list ./... | grep -v /test/) -coverprofile cover.out > /dev/null 2>&1
Expand Down Expand Up @@ -399,30 +402,30 @@ generate-all: generate generate-deploy bundle
@$(MAKE) fmt

.PHONY: test-e2e # You will need to have a Minikube/Kind cluster up and running to run this target, and run container-builder before the test
label = "flows-non-persistence" # possible values are flows-non-persistence, flows-persistence, platform, cluster
label = "flows-ephemeral" # possible values are flows-ephemeral, flows-persistence, platform, cluster
test-e2e:
ifeq ($(label), cluster)
@echo "🌐 Running e2e tests for cluster..."
go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/clusterplatform_test.go \
-v -ginkgo.v -ginkgo.no-color -ginkgo.github-output -ginkgo.label-filter=$(label) \
-ginkgo.junit-report=./e2e-test-report-clusterplatform_test.xml -timeout 60m;
-ginkgo.junit-report=./e2e-test-report-clusterplatform_test.xml -timeout 60m KUSTOMIZE=$(KUSTOMIZE);
else ifeq ($(label), platform)
@echo "📦 Running e2e tests for platform..."
go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/platform_test.go \
-v -ginkgo.v -ginkgo.no-color -ginkgo.github-output -ginkgo.label-filter=$(label) \
-ginkgo.junit-report=./e2e-test-report-platform_test.xml -timeout 60m;
else ifeq ($(label), flows-non-persistence)
@echo "🔄 Running e2e tests for flows-non-persistence..."
-ginkgo.junit-report=./e2e-test-report-platform_test.xml -timeout 60m KUSTOMIZE=$(KUSTOMIZE);
else ifeq ($(label), flows-ephemeral)
@echo "🔄 Running e2e tests for flows-ephemeral..."
go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/workflow_test.go \
-v -ginkgo.v -ginkgo.no-color -ginkgo.github-output -ginkgo.label-filter=$(label) \
-ginkgo.junit-report=./e2e-test-report-workflow_test.xml -timeout 60m;
-ginkgo.junit-report=./e2e-test-report-workflow_test.xml -timeout 60m KUSTOMIZE=$(KUSTOMIZE);
else ifeq ($(label), flows-persistence)
@echo "🔁 Running e2e tests for flows-persistence..."
go test ./test/e2e/e2e_suite_test.go ./test/e2e/helpers.go ./test/e2e/workflow_test.go \
-v -ginkgo.v -ginkgo.no-color -ginkgo.github-output -ginkgo.label-filter=$(label) \
-ginkgo.junit-report=./e2e-test-report-workflow_test.xml -timeout 60m;
-ginkgo.junit-report=./e2e-test-report-workflow_test.xml -timeout 60m KUSTOMIZE=$(KUSTOMIZE);
else
@echo "❌ Invalid label. Please use one of: cluster, platform, flows-non-persistence, flows-persistence"
@echo "❌ Invalid label. Please use one of: cluster, platform, flows-ephemeral, flows-persistence"
endif


Expand Down
12 changes: 9 additions & 3 deletions hack/local/run-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
# You must have minikube installed
MINIKUBE_PROFILE=${1:-minikube}
SKIP_IMG_BUILD=${2:-false}
TEST_LABELS=${3:-"flows-non-persistence"} # possible values are flows-non-persistence, flows-persistence, platform, cluster
TEST_LABELS=${3:-"flows-ephemeral"} # possible values are flows-ephemeral, flows-persistence, platform, cluster
SKIP_UNDEPLOY=${4:-false}

# Emoticons and enhanced messages
echo "🚀 Using minikube profile ${MINIKUBE_PROFILE}"
Expand All @@ -37,8 +38,13 @@ fi
# clean up previous runs, hiding logs
echo "🧹 Cleaning up previous test namespaces..."
kubectl get namespaces -o name | awk -F/ '/^namespace\/test/ {print $2}' | xargs kubectl delete namespace &> /dev/null
echo "🧹 Undeploying previous instances..."
make undeploy ignore-not-found=true &> /dev/null

if [ "${SKIP_UNDEPLOY}" = false ]; then
echo "🧹 Cleaning up previous test resources namespace..."
kubectl delete namespace e2e-resources &> /dev/null
echo "🧹 Undeploying previous instances..."
make undeploy ignore-not-found=true &> /dev/null
fi

# Image build process
if [ "${SKIP_IMG_BUILD}" = "false" ]; then
Expand Down
2 changes: 1 addition & 1 deletion internal/controller/platform/platformutils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const dockerFile = "FROM docker.io/apache/default-test-kie-sonataflow-builder:ma

func TestSonataFlowBuildController(t *testing.T) {
platform := test.GetBasePlatform()
dockerfileBytes, err := os.ReadFile("../../../test/builder/Dockerfile")
dockerfileBytes, err := os.ReadFile("testdata/platformTest.Dockerfile")
if err != nil {
assert.Fail(t, "Unable to read base Dockerfile")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ USER 1001
WORKDIR /home/kogito/kogito-base

# Copy from build context to skeleton resources project
COPY --chown=1001 . ./src/main/resources
COPY --chown=1001 ../../../../test/builder ./src/main/resources

# Maven vars enhirited from the base image
RUN ${MAVEN_HOME}/bin/mvn -U -B ${MAVEN_ARGS_APPEND} -s ${MAVEN_SETTINGS_PATH} clean install -DskipTests
Expand Down
67 changes: 0 additions & 67 deletions test/builder/greetings.sw.json

This file was deleted.

53 changes: 35 additions & 18 deletions test/e2e/clusterplatform_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,15 +73,20 @@ var _ = Describe("Cluster Platform Use Cases :: ", Label("cluster"), Ordered, fu
Expect(err).NotTo(HaveOccurred())
}
}

// Remove SonataFlowClusterPlatform created in the last run
cmd := exec.Command("kubectl", "delete", "sonataflowclusterplatform", "--all", "--wait")
_, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())
})
var _ = Context("with supporting services enabled", func() {
DescribeTable("against a platform in a separate namespace", func(testcaseDir string, profile string, persistenceType string, withServices bool) {
By("Deploy the SonataFlowPlatform CR")
var manifests []byte
EventuallyWithOffset(1, func() error {
var err error
cmd := exec.Command("kubectl", "kustomize", filepath.Join(projectDir,
test.GetSonataFlowE2EPlatformServicesDirectory(), profile, clusterWideEphemeral))
cmd := exec.Command("kubectl", "kustomize",
test.GetPathFromE2EDirectory("platform", "services", profile, clusterWideEphemeral))
manifests, err = utils.Run(cmd)
return err
}, time.Minute, time.Second).Should(Succeed())
Expand All @@ -91,7 +96,7 @@ var _ = Describe("Cluster Platform Use Cases :: ", Label("cluster"), Ordered, fu
_, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())

By("Wait for SonatatFlowPlatform CR in " + targetNamespace + " to be ready")
By("Wait for SonataFlowPlatform CR in " + targetNamespace + " to be ready")
// wait for platform to be ready
EventuallyWithOffset(1, func() error {
cmd = exec.Command("kubectl", "wait", "sfplatform", "-n", targetNamespace, "sonataflow-platform", "--for", "condition=Succeed", "--timeout=5s")
Expand All @@ -116,7 +121,7 @@ var _ = Describe("Cluster Platform Use Cases :: ", Label("cluster"), Ordered, fu
}, 20*time.Minute, 5).Should(Succeed())

if withServices {
By("Deploy SonatatFlowPlatform CR with services configured in " + targetNamespace2)
By("Deploy SonataFlowPlatform CR with services configured in " + targetNamespace2)
EventuallyWithOffset(1, func() error {
var err error
cmd := exec.Command("kubectl", "kustomize", filepath.Join(projectDir,
Expand Down Expand Up @@ -148,18 +153,24 @@ var _ = Describe("Cluster Platform Use Cases :: ", Label("cluster"), Ordered, fu
return returnedValue
}, 20*time.Minute, 5).Should(Equal([]byte("''")))

By("Replacing the image with a prebuilt one and rollout")
EventuallyWithOffset(1, func() error {
return kubectlPatchSonataFlowImageAndRollout(targetNamespace2, prebuiltWorkflows.CallBack.Name, prebuiltWorkflows.CallBack.Tag)
}, 3*time.Minute, time.Second).Should(Succeed())

By("Waiting for Data Index and Jobs Service URLs")
dataIndexServiceUrl := services.GenerateServiceURL(constants.DefaultHTTPProtocol, targetNamespace2, "sonataflow-platform-"+constants.DataIndexServiceName)
jobServiceUrl := services.GenerateServiceURL(constants.DefaultHTTPProtocol, targetNamespace2, "sonataflow-platform-"+constants.JobServiceName)
EventuallyWithOffset(1, func() []byte {
cmd = exec.Command("kubectl", "get", "sf", "-n", targetNamespace2, "callbackstatetimeouts", "-o", "jsonpath='{.status.services.dataIndexRef.url}'")
cmd = exec.Command("kubectl", "get", "sf", "-n", targetNamespace2, prebuiltWorkflows.CallBack.Name, "-o", "jsonpath='{.status.services.dataIndexRef.url}'")
returnedValue, _ := utils.Run(cmd)
return returnedValue
}, 20*time.Minute, 5).Should(Equal([]byte("'" + dataIndexServiceUrl + "'")))
}, 3*time.Minute, 5).Should(Equal([]byte("'" + dataIndexServiceUrl + "'")))
EventuallyWithOffset(1, func() []byte {
cmd = exec.Command("kubectl", "get", "sf", "-n", targetNamespace2, "callbackstatetimeouts", "-o", "jsonpath='{.status.services.jobServiceRef.url}'")
cmd = exec.Command("kubectl", "get", "sf", "-n", targetNamespace2, prebuiltWorkflows.CallBack.Name, "-o", "jsonpath='{.status.services.jobServiceRef.url}'")
returnedValue, _ := utils.Run(cmd)
return returnedValue
}, 20*time.Minute, 5).Should(Equal([]byte("'" + jobServiceUrl + "'")))
}, 3*time.Minute, 5).Should(Equal([]byte("'" + jobServiceUrl + "'")))
} else {
EventuallyWithOffset(1, func() error {
var err error
Expand Down Expand Up @@ -193,32 +204,38 @@ var _ = Describe("Cluster Platform Use Cases :: ", Label("cluster"), Ordered, fu
returnedValue, _ := utils.Run(cmd)
return returnedValue
}, 20*time.Minute, 5).Should(Equal([]byte("'" + jobServiceUrl + "'")))

By("Replacing the image with a prebuilt one and rollout")
EventuallyWithOffset(1, func() error {
return kubectlPatchSonataFlowImageAndRollout(targetNamespace2, prebuiltWorkflows.CallBack.Name, prebuiltWorkflows.CallBack.Tag)
}, 3*time.Minute, time.Second).Should(Succeed())

By("Waiting for Data Index and Jobs Service URLs")
EventuallyWithOffset(1, func() []byte {
cmd = exec.Command("kubectl", "get", "sf", "-n", targetNamespace2, "callbackstatetimeouts", "-o", "jsonpath='{.status.services.dataIndexRef.url}'")
cmd = exec.Command("kubectl", "get", "sf", "-n", targetNamespace2, prebuiltWorkflows.CallBack.Name, "-o", "jsonpath='{.status.services.dataIndexRef.url}'")
returnedValue, _ := utils.Run(cmd)
return returnedValue
}, 20*time.Minute, 5).Should(Equal([]byte("'" + dataIndexServiceUrl + "'")))
}, 3*time.Minute, 5).Should(Equal([]byte("'" + dataIndexServiceUrl + "'")))
EventuallyWithOffset(1, func() []byte {
cmd = exec.Command("kubectl", "get", "sf", "-n", targetNamespace2, "callbackstatetimeouts", "-o", "jsonpath='{.status.services.jobServiceRef.url}'")
cmd = exec.Command("kubectl", "get", "sf", "-n", targetNamespace2, prebuiltWorkflows.CallBack.Name, "-o", "jsonpath='{.status.services.jobServiceRef.url}'")
returnedValue, _ := utils.Run(cmd)
return returnedValue
}, 20*time.Minute, 5).Should(Equal([]byte("'" + jobServiceUrl + "'")))
}, 3*time.Minute, 5).Should(Equal([]byte("'" + jobServiceUrl + "'")))
}
cmd = exec.Command("kubectl", "delete", "SonataFlowClusterPlatform", "cluster", "--wait")
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())
},
Entry("without services configured", test.GetSonataFlowE2EPlatformNoServicesDirectory(), metadata.PreviewProfile.String(), ephemeral, false),
Entry("with services configured", test.GetSonataFlowE2EPlatformServicesDirectory(), metadata.PreviewProfile.String(), "ephemeral-with-workflow", true),
Entry("without services configured", test.GetPathFromE2EDirectory("platform", "noservices"), metadata.GitOpsProfile.String(), ephemeral, false),
Entry("with services configured", test.GetPathFromE2EDirectory("platform", "services"), metadata.GitOpsProfile.String(), "ephemeral-with-workflow", true),
)

DescribeTable("against a platform in a separate namespace", func(testcaseDir string, profile string, persistenceType string) {
By("Deploy the SonataFlowPlatform CR")
var manifests []byte
EventuallyWithOffset(1, func() error {
var err error
cmd := exec.Command("kubectl", "kustomize", filepath.Join(projectDir,
test.GetSonataFlowE2EPlatformServicesDirectory(), profile, clusterWideEphemeral))
cmd := exec.Command("kubectl", "kustomize", test.GetPathFromE2EDirectory("platform", "services", profile, clusterWideEphemeral))
manifests, err = utils.Run(cmd)
return err
}, time.Minute, time.Second).Should(Succeed())
Expand Down Expand Up @@ -287,8 +304,8 @@ var _ = Describe("Cluster Platform Use Cases :: ", Label("cluster"), Ordered, fu
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())
},
Entry("with only Data Index configured", test.GetSonataFlowE2EPlatformServicesDirectory(), metadata.PreviewProfile.String(), ephemeralDataIndex),
Entry("with only Job Service configured", test.GetSonataFlowE2EPlatformServicesDirectory(), metadata.PreviewProfile.String(), ephemeralJobService),
Entry("with only Data Index configured", test.GetPathFromE2EDirectory("platform", "services"), metadata.GitOpsProfile.String(), ephemeralDataIndex),
Entry("with only Job Service configured", test.GetPathFromE2EDirectory("platform", "services"), metadata.GitOpsProfile.String(), ephemeralJobService),
)
})
})
Loading

0 comments on commit bab8c7c

Please sign in to comment.