chore: bump deps, refactor Py tests to use dict options

.github/workflows/test.yml

@@ -14,7 +14,6 @@ jobs:
  strategy:
  matrix:
  python-version:
- - '3.8'
  - '3.9'
  - '3.10'
  - '3.11'
@@ -42,6 +41,8 @@ jobs:
  python-version: '${{ matrix.python-version }}'
  cache: pip
  - name: Install Python test dependencies
- run: pip install -r src/test/python/requirements.txt
+ run: |
+ pip install --upgrade pip
+ pip install -r src/test/python/requirements.txt
  - name: Run Python tests
  run: pytest

.releaserc

@@ -68,73 +68,6 @@
  ],
  "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
  }
- ],
- [
- "@semantic-release/release-notes-generator",
- {
- "preset": "conventionalcommits",
- "parserOpts": {
- "noteKeywords": [
- "BREAKING CHANGE",
- "BREAKING CHANGES",
- "BREAKING"
- ]
- },
- "writerOpts": {
- "commitsSort": [
- "subject",
- "scope"
- ]
- },
- "presetConfig": {
- "types": [
- {
- "type": "feat",
- "section": "🌞 Features"
- },
- {
- "type": "fix",
- "section": "🐛 Bug Fixes"
- },
- {
- "type": "perf",
- "section": "🚀 Performance Improvements"
- },
- {
- "type": "revert",
- "section": "⏩ Reverts"
- },
- {
- "type": "docs",
- "section": "📝 Documentation"
- },
- {
- "type": "style",
- "section": "🎨 Styles"
- },
- {
- "type": "refactor",
- "section": "🧑‍💻 Code Refactoring"
- },
- {
- "type": "test",
- "section": "✅ Tests"
- },
- {
- "type": "build",
- "section": "🤖 Build System"
- },
- {
- "type": "ci",
- "section": "🔁 Continuous Integration"
- },
- {
- "type": "chore",
- "section": "🧹 Chores"
- }
- ]
- }
- }
  ]
  ]
 }

README.md

@@ -192,7 +192,8 @@ You can use the connector as a library in Databricks to ingest data into Qdrant.
 - Select `Install New` to open the library installation modal.
 - Search for `io.qdrant:spark:VERSION` in the Maven packages and click `Install`.
 
-<img width="1064" alt="Screenshot 2024-01-05 at 17 20 01 (1)" src="https://github.com/qdrant/qdrant-spark/assets/46051506/d95773e0-c5c6-4ff2-bf50-8055bb08fd1b">
+<img width="704" alt="Screenshot 2024-04-28 at 11 34 17 AM" src="https://github.com/qdrant/qdrant-spark/assets/46051506/0c1bd356-3fba-436a-90ce-d8ff39b02d1f">
+
 
 ## Datatype support
 

pom.xml

@@ -39,12 +39,12 @@
  <dependency>
  <groupId>io.qdrant</groupId>
  <artifactId>client</artifactId>
- <version>1.8.0</version>
+ <version>1.9.0</version>
  </dependency>
  <dependency>
  <groupId>com.google.guava</groupId>
  <artifactId>guava</artifactId>
- <version>32.0.0-jre</version>
+ <version>33.1.0-jre</version>
  </dependency>
  <dependency>
  <groupId>io.grpc</groupId>
@@ -56,7 +56,7 @@
  <dependency>
  <groupId>org.slf4j</groupId>
  <artifactId>slf4j-api</artifactId>
- <version>2.0.7</version>
+ <version>2.0.13</version>
  </dependency>
  <dependency>
  <groupId>org.apache.spark</groupId>
@@ -75,13 +75,13 @@
  <dependency>
  <groupId>org.testcontainers</groupId>
  <artifactId>qdrant</artifactId>
- <version>1.19.6</version>
+ <version>1.19.7</version>
  <scope>test</scope>
  </dependency>
  <dependency>
  <groupId>org.testcontainers</groupId>
  <artifactId>junit-jupiter</artifactId>
- <version>1.19.4</version>
+ <version>1.19.7</version>
  <scope>test</scope>
  </dependency>
  </dependencies>

src/test/python/conftest.py

@@ -1,24 +1,26 @@
 import pytest
-from testcontainers.core.container import DockerContainer # type: ignore
-from testcontainers.core.waiting_utils import wait_for_logs # type: ignore
+from testcontainers.qdrant import QdrantContainer
 from qdrant_client import QdrantClient, models
 import uuid
 from pyspark.sql import SparkSession
 from typing import NamedTuple
+from uuid import uuid4
 
 
 QDRANT_GRPC_PORT = 6334
 QDRANT_EMBEDDING_DIM = 6
 QDRANT_DISTANCE = models.Distance.COSINE
+QDRANT_API_KEY = uuid4().hex
 
 
 class Qdrant(NamedTuple):
  url: str
+ api_key: str
  collection_name: str
  client: QdrantClient
 
 
-qdrant_container = DockerContainer("qdrant/qdrant").with_exposed_ports(QDRANT_GRPC_PORT)
+qdrant_container = QdrantContainer(image="qdrant/qdrant:latest", api_key=QDRANT_API_KEY)
 
 
 # Reference: https://gist.github.com/dizzythinks/f3bb37fd8ab1484bfec79d39ad8a92d3
@@ -36,9 +38,6 @@ def get_pom_version():
 @pytest.fixture(scope="module", autouse=True)
 def setup_container(request):
  qdrant_container.start()
- wait_for_logs(
- qdrant_container, ".*Actix runtime found; starting in Actix runtime.*", 60
- )
 
  def remove_container():
  qdrant_container.stop()
@@ -70,6 +69,8 @@ def qdrant():
  host=host,
  grpc_port=grpc_port,
  prefer_grpc=True,
+ api_key=QDRANT_API_KEY,
+ https=False,
  )
 
  collection_name = str(uuid.uuid4())
@@ -99,6 +100,7 @@ def qdrant():
  url=f"http://{host}:{grpc_port}",
  client=client,
  collection_name=collection_name,
+ api_key=QDRANT_API_KEY,
  )
 
  return client.close()

src/test/python/requirements.txt

@@ -1,4 +1,4 @@
 pyspark==3.5.1
-pytest==8.0.2
-qdrant-client==1.7.3
-testcontainers==3.7.1
+pytest==8.2.0
+qdrant-client==1.9.0
+testcontainers==4.4.0