Skip to content

Commit

Permalink
test: Add Karate test job to pipeline (#30486) (#30708)
Browse files Browse the repository at this point in the history
### Proposed Changes
* Add baseline Karate project following pattern of postman in a separate
maven module
  • Loading branch information
spbolton authored Nov 20, 2024
1 parent ad7609b commit 3c2c587
Show file tree
Hide file tree
Showing 19 changed files with 578 additions and 24 deletions.
2 changes: 1 addition & 1 deletion .github/actions/core-cicd/maven-job/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -321,4 +321,4 @@ runs:
name: "build-reports-test-${{ inputs.stage-name }}"
path: |
**/target/jacoco-report/*.exec
**/target/*-reports/TEST-*.xml
**/target/*-reports/*.xml
1 change: 1 addition & 0 deletions .github/filters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ backend: &backend
- 'core-web/pom.xml'
- 'dotCMS/src/main/webapp/html/**/!(*.{css,js})'
- 'dotcms-postman/**'
- 'test-karate/**'
- 'e2e/**'
- 'dotCMS/!(src/main/webapp/html/)**'
- 'dotcms-integration/**'
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/cicd_1-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ jobs:
jvm_unit_test: ${{ needs.initialize.outputs.jvm_unit_test == 'true' }}
integration: ${{ needs.initialize.outputs.backend == 'true' }}
postman: ${{ needs.initialize.outputs.backend == 'true' }}
karate: ${{ needs.initialize.outputs.backend == 'true' }}
frontend: ${{ needs.initialize.outputs.frontend == 'true' }}
cli: ${{ needs.initialize.outputs.cli == 'true' }}
e2e: ${{ needs.initialize.outputs.build == 'true' }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/cicd_2-merge-queue.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ jobs:
jvm_unit_test: ${{ needs.initialize.outputs.jvm_unit_test == 'true' }}
integration: ${{ needs.initialize.outputs.backend == 'true' }}
postman: ${{ needs.initialize.outputs.backend == 'true' }}
karate: ${{ needs.initialize.outputs.backend == 'true' }}
frontend: ${{ needs.initialize.outputs.frontend == 'true' }}
cli: ${{ needs.initialize.outputs.cli == 'true' }}
e2e: ${{ needs.initialize.outputs.build == 'true' }}
Expand Down
37 changes: 36 additions & 1 deletion .github/workflows/cicd_comp_test-phase.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ on:
required: false
type: boolean
default: false
karate:
required: false
type: boolean
default: false
integration:
required: false
type: boolean
Expand Down Expand Up @@ -180,7 +184,38 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
artifacts-from: ${{ env.ARTIFACT_RUN_ID }}
cleanup-runner: true

# Karate Tests
karate-tests:
name: Karate Tests - ${{ matrix.suites.name }}
runs-on: ubuntu-24.04
if: inputs.karate || inputs.run-all-tests
strategy:
fail-fast: false
matrix:
suites:
- { name: "Default", pathName: "default", tests: 'KarateCITests#defaults' }
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: ./.github/actions/core-cicd/maven-job
with:
stage-name: "Karate ${{ matrix.suites.name }}"
maven-args: "verify -pl :dotcms-test-karate -Dkarate.test.skip=false -Dit.test=${{ matrix.suites.tests }}"
generates-test-results: true
dotcms-license: ${{ secrets.DOTCMS_LICENSE }}
requires-node: true
needs-docker-image: true
github-token: ${{ secrets.GITHUB_TOKEN }}
artifacts-from: ${{ env.ARTIFACT_RUN_ID }}
cleanup-runner: true
- id: upload-karate-report
name: Upload Karate-report
uses: actions/upload-artifact@v4
with:
name: karate-reports
path: dotCMS/target/karate-reports
# E2E Tests
linux-e2e-tests:
name: E2E Tests ${{matrix.suites.name}}
Expand Down
20 changes: 0 additions & 20 deletions e2e/dotcms-e2e-java/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -102,26 +102,6 @@
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-suite</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-suite</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.microsoft.playwright</groupId>
<artifactId>playwright</artifactId>
Expand Down
13 changes: 11 additions & 2 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ build-prod:

# Runs a comprehensive test suite including core integration and postman tests, suitable for final validation
build-test-full:
./mvnw clean install -Dcoreit.test.skip=false -Dpostman.test.skip=false
./mvnw clean install -Dcoreit.test.skip=false -Dpostman.test.skip=false -Dkarate.test.skip=false

# Builds a specified module without its dependencies, defaulting to the core server (dotcms-core)
build-select-module module="dotcms-core":
Expand Down Expand Up @@ -99,6 +99,9 @@ dev-tomcat-stop:
test-postman collections='page':
./mvnw -pl :dotcms-postman verify -Dpostman.test.skip=false -Pdebug -Dpostman.collections={{ collections }}

test-karate collections='KarateCITests#defaults':
./mvnw -pl :dotcms-test-karate verify -Dkarate.test.skip=false -Pdebug -Dit.test={{ collections }}

# Stops Postman-related Docker containers
postman-stop:
./mvnw -pl :dotcms-postman -Pdocker-stop -Dpostman.test.skip=false
Expand All @@ -121,7 +124,13 @@ build-core-only:

# Prepares the environment for running integration tests in an IDE
test-integration-ide:
./mvnw -pl :dotcms-integration pre-integration-test -Dcoreit.test.skip=false
./mvnw -pl :dotcms-integration pre-integration-test -Dcoreit.test.skip=false -Dtomcat.port=8080

test-postman-ide:
./mvnw -pl :dotcms-test-karate pre-integration-test -Dpostman.test.skip=false -Dtomcat.port=8080

test-karate-ide:
./mvnw -pl :dotcms-test-karate pre-integration-test -Dkarate.test.skip=false -Dtomcat.port=8080

# Stops integration test services
test-integration-stop:
Expand Down
1 change: 1 addition & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
<module>reports</module>
<module>e2e/dotcms-e2e-java</module>
<module>e2e/dotcms-e2e-node</module>
<module>test-karate</module>
</modules>
<repositories>
<repository>
Expand Down
203 changes: 203 additions & 0 deletions test-karate/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.dotcms</groupId>
<artifactId>dotcms-parent</artifactId>
<version>${revision}${sha1}${changelist}</version>
<relativePath>../parent/pom.xml</relativePath>
</parent>

<packaging>jar</packaging>
<artifactId>dotcms-test-karate</artifactId>

<properties>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<maven.compiler.release>21</maven.compiler.release>

<license.use>true</license.use>
<clean.docker.volumes>true</clean.docker.volumes>
<cleanup.before.tests>true</cleanup.before.tests>
<postman.test.skip>true</postman.test.skip>
<!-- Some tests e.g. in ApiToken_Resource.postman_collection currently fail unless docker external port is same as internal -->
<tomcat.port>8080</tomcat.port>
<docker.jacoco.skip>false</docker.jacoco.skip>
<karate.version>1.5.0</karate.version>
<karate.test.skip>true</karate.test.skip>
<karate.base.url>http://localhost:${tomcat.port}</karate.base.url>
<it.test>KarateCITests#defaults</it.test>
</properties>

<build>
<testResources>
<testResource>
<directory>src/test/java</directory>
<excludes>
<exclude>**/*.java</exclude>
</excludes>
</testResource>
</testResources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>io.fabric8</groupId>
<artifactId>docker-maven-plugin</artifactId>
<configuration>
<skip>${karate.test.skip}</skip>
<showLogs>true</showLogs>
<follow>true</follow>
<imagesMap>
<dotcms>
<run>
<env>
<CATALINA_OPTS>-XX:+PrintFlagsFinal</CATALINA_OPTS>
<DB_MAX_TOTAL>15</DB_MAX_TOTAL>
<DOT_INDEX_POLICY_SINGLE_CONTENT>FORCE</DOT_INDEX_POLICY_SINGLE_CONTENT>
<DOT_ASYNC_REINDEX_COMMIT_LISTENERS>false</DOT_ASYNC_REINDEX_COMMIT_LISTENERS>
<DOT_ASYNC_COMMIT_LISTENERS>false</DOT_ASYNC_COMMIT_LISTENERS>
<DOT_CACHE_GRAPHQLQUERYCACHE_SECONDS>600</DOT_CACHE_GRAPHQLQUERYCACHE_SECONDS>
<JVM_ENDPOINT_TEST_PASS>obfuscate_me</JVM_ENDPOINT_TEST_PASS>
<DOT_ENABLE_SCRIPTING>true</DOT_ENABLE_SCRIPTING>
<DOT_ANNOUNCEMENTS_BASE_URL>http://localhost:${tomcat.port}</DOT_ANNOUNCEMENTS_BASE_URL>
<DOT_ALLOW_ACCESS_TO_PRIVATE_SUBNETS>true</DOT_ALLOW_ACCESS_TO_PRIVATE_SUBNETS>
<DOT_DOTCMS_DEV_MODE>true</DOT_DOTCMS_DEV_MODE>
<DOT_AI_API_URL>http://wm:8080/c</DOT_AI_API_URL>
<DOT_AI_IMAGE_API_URL>http://wm:${tomcat.port}/i</DOT_AI_IMAGE_API_URL>
<DOT_AI_EMBEDDINGS_API_URL>http://wm:${tomcat.port}/e</DOT_AI_EMBEDDINGS_API_URL>
<DOT_AI_MODELS_API_URL>http://wm:${tomcat.port}/m</DOT_AI_MODELS_API_URL>
</env>
</run>
</dotcms>
</imagesMap>
</configuration>
<executions>
<execution>
<id>cleanup-at-start</id>
<goals>
<goal>stop</goal>
<goal>volume-remove</goal>
</goals>
<phase>pre-integration-test</phase>
</execution>
<execution>
<id>start</id>
<goals>
<goal>start</goal>
</goals>
<phase>pre-integration-test</phase>
</execution>
<execution>
<id>stop</id>
<goals>
<goal>stop</goal>
</goals>
<!-- should be post-integration test but need to make sure it does not stop
before dump of jacoco -->
<phase>verify</phase>
</execution>
</executions>
</plugin>

<!-- Maven Failsafe Plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>3.0.0</version>
<configuration>
<skip>${karate.test.skip}</skip>
<systemPropertyVariables>
<karate.base.url>${karate.base.url}</karate.base.url>
</systemPropertyVariables>
<includes>
<include>**/*.java</include>
</includes>
<testFailureIgnore>false</testFailureIgnore>
</configuration>
<executions>
<execution>
<id>integration-tests</id>
<phase>integration-test</phase>
<goals>
<goal>integration-test</goal>
</goals>
</execution>
<execution>
<id>verify-tests</id>
<phase>verify</phase>
<goals>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>

</plugins>
</build>
<dependencies>
<!-- <dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>
-->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
<version>5.10.2</version>
</dependency>
<dependency>
<groupId>io.karatelabs</groupId>
<artifactId>karate-junit5</artifactId>
<version>${karate.version}</version>
<scope>test</scope>
<exclusions>
<!--
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</exclusion>
-->
</exclusions>
</dependency>
</dependencies>

</project>
20 changes: 20 additions & 0 deletions test-karate/src/test/java/KarateCITests.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import static org.junit.jupiter.api.Assertions.assertEquals;

import com.intuit.karate.Results;
import com.intuit.karate.Runner;
import org.junit.jupiter.api.Test;

public class KarateCITests {
// Note this is a JUnit 5 test compare with @Karate.Test annotated methods.https://stackoverflow.com/questions/65577487/whats-the-purpose-of-karate-junit5-when-you-can-run-tests-without-it

@Test
void defaults() {
Results results = Runner.path("classpath:tests/defaults").tags("~@ignore")
.outputHtmlReport(true)
.outputJunitXml(true)
.outputCucumberJson(true)
.parallel(1);
assertEquals(0, results.getFailCount(), results.getErrorMessages());
}

}
Loading

0 comments on commit 3c2c587

Please sign in to comment.