-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* test: Pytest * ci: renamed workflow * ci: release JAR * ci: YAML formatting
- Loading branch information
Showing
12 changed files
with
1,629 additions
and
72 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,24 +1,39 @@ | ||
| name: Maven Tests | ||
| name: Maven and Python Tests | ||
| "on": | ||
| pull_request: | ||
| types: | ||
| - opened | ||
| - edited | ||
| - synchronize | ||
| - reopened | ||
| - opened | ||
| - edited | ||
| - synchronize | ||
| - reopened | ||
| env: | ||
| QDRANT_URL: "${{ secrets.QDRANT_URL }}" | ||
| QDRANT_API_KEY: "${{ secrets.QDRANT_API_KEY }}" | ||
| jobs: | ||
| test: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - uses: actions/setup-java@v3 | ||
| with: | ||
| java-version: "8" | ||
| distribution: temurin | ||
| - name: Run the Maven tests | ||
| run: mvn test | ||
| - name: Generate assembly fat JAR | ||
| run: mvn clean package | ||
| - uses: actions/checkout@v4 | ||
| - uses: actions/setup-java@v3 | ||
| with: | ||
| java-version: "8" | ||
| distribution: temurin | ||
| - name: Cache local Maven repository | ||
| uses: actions/cache@v3 | ||
| with: | ||
| path: ~/.m2/repository | ||
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-maven- | ||
| - name: Run Maven tests | ||
| run: mvn test | ||
| - name: Generate JARs | ||
| run: mvn clean package -DskipTests | ||
| - uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: '3.11' | ||
| cache: 'pip' | ||
| - name: Install Python test dependencies | ||
| run: pip install -r src/test/python/requirements.txt | ||
| - name: Run Python tests | ||
| run: pytest |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| name: Build and release JAR files | ||
|
|
||
| on: | ||
| release: | ||
| types: | ||
| - published | ||
|
|
||
| jobs: | ||
| upload-jar: | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| contents: write # release changes require contents write | ||
|
|
||
| steps: | ||
| - name: Check out code | ||
| uses: actions/checkout@v4 | ||
|
|
||
| - uses: actions/setup-java@v3 | ||
| with: | ||
| java-version: "8" | ||
| distribution: temurin | ||
|
|
||
| - name: Generate JARs | ||
| run: mvn clean package -DskipTests | ||
|
|
||
| - name: Set project version env variable | ||
| run: | | ||
| echo "PROJECT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)" >> $GITHUB_ENV | ||
| - name: Build and upload JAR | ||
| env: | ||
| GH_TOKEN: ${{ github.token }} | ||
| run: | | ||
| gh release upload ${{ github.event.release.tag_name }} target/spark-${{ env.PROJECT_VERSION }}.jar |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| import pytest | ||
| from testcontainers.core.container import DockerContainer # type: ignore | ||
| from testcontainers.core.waiting_utils import wait_for_logs # type: ignore | ||
| from qdrant_client import QdrantClient, models | ||
| import uuid | ||
| from pyspark.sql import SparkSession | ||
| from typing import NamedTuple | ||
|
|
||
|
|
||
| QDRANT_GRPC_PORT = 6334 | ||
| QDRANT_EMBEDDING_DIM = 6 | ||
| QDRANT_DISTANCE = models.Distance.COSINE | ||
|
|
||
|
|
||
| class Qdrant(NamedTuple): | ||
| url: str | ||
| collection_name: str | ||
| client: QdrantClient | ||
|
|
||
|
|
||
| qdrant_container = DockerContainer("qdrant/qdrant").with_exposed_ports(QDRANT_GRPC_PORT) | ||
|
|
||
|
|
||
| # Reference: https://gist.github.com/dizzythinks/f3bb37fd8ab1484bfec79d39ad8a92d3 | ||
| def get_pom_version(): | ||
| from xml.etree import ElementTree as et | ||
|
|
||
| ns = "http://maven.apache.org/POM/4.0.0" | ||
| et.register_namespace("", ns) | ||
| tree = et.ElementTree() | ||
| tree.parse("pom.xml") | ||
| p = tree.getroot().find("{%s}version" % ns) | ||
| return p.text | ||
|
|
||
|
|
||
| @pytest.fixture(scope="module", autouse=True) | ||
| def setup_container(request): | ||
| qdrant_container.start() | ||
| wait_for_logs( | ||
| qdrant_container, ".*Actix runtime found; starting in Actix runtime.*", 60 | ||
| ) | ||
|
|
||
| def remove_container(): | ||
| qdrant_container.stop() | ||
|
|
||
| request.addfinalizer(remove_container) | ||
|
|
||
|
|
||
| @pytest.fixture(scope="session") | ||
| def spark_session(): | ||
| spark_session = ( | ||
| SparkSession.builder.config( | ||
| "spark.jars", f"target/spark-{get_pom_version()}.jar" | ||
| ) | ||
| .master("local[*]") | ||
| .appName("qdrant") | ||
| .getOrCreate() | ||
| ) | ||
|
|
||
| return spark_session | ||
|
|
||
|
|
||
| @pytest.fixture() | ||
| def qdrant() -> Qdrant: | ||
| host = qdrant_container.get_container_host_ip() | ||
| grpc_port = qdrant_container.get_exposed_port(QDRANT_GRPC_PORT) | ||
|
|
||
| client = QdrantClient( | ||
| host=host, | ||
| grpc_port=grpc_port, | ||
| prefer_grpc=True, | ||
| ) | ||
|
|
||
| collection_name = str(uuid.uuid4()) | ||
| client.create_collection( | ||
| collection_name=collection_name, | ||
| vectors_config={ | ||
| "dense": models.VectorParams( | ||
| size=QDRANT_EMBEDDING_DIM, | ||
| distance=QDRANT_DISTANCE, | ||
| ), | ||
| "": models.VectorParams( | ||
| size=QDRANT_EMBEDDING_DIM, | ||
| distance=QDRANT_DISTANCE, | ||
| ), | ||
| }, | ||
| sparse_vectors_config={ | ||
| "sparse": models.SparseVectorParams(), | ||
| }, | ||
| ) | ||
|
|
||
| return Qdrant( | ||
| url=f"http://{host}:{grpc_port}", | ||
| client=client, | ||
| collection_name=collection_name, | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| pyspark==3.5.1 | ||
| pytest==8.0.2 | ||
| qdrant-client==1.7.3 | ||
| testcontainers==3.7.1 |
Oops, something went wrong.