Merge pull request #133 from stardog-union/release/0.13.0

Release 0.13.0
stardog-union · Jan 11, 2023 · e2d7bb0 · e2d7bb0
2 parents f51bfef + 73e6edb
commit e2d7bb0
Show file tree

Hide file tree

Showing 38 changed files with 2,031 additions and 1,855 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,63 +1,212 @@
+---
 version: 2.1
 
+installs_supported_python_versions: &installs_supported_python_versions
+  run:
+    name: Install supported python versions
+    no_output_timeout: 15m
+    command: |
+      set -x
+      sudo apt-get update -y
+      yes | sudo add-apt-repository ppa:deadsnakes/ppa || true
+      sudo apt-get install python3.{8,9,10,11} -yq
+      sudo apt-get install python3.{8,9}-distutils -yq
+
+run_basic_admin_suite: &run_basic_admin_suite
+  run:
+    name: Runs the basic admin suite
+    no_output_timeout: 15m
+    command: |
+      set -x
+      pytest test/test_admin_basic.py -s
+
+run_connection_suite: &run_connection_suite
+  run:
+    name: Runs the test_connection suite
+    no_output_timeout: 15m
+    command: |
+      set -x
+      pytest test/test_connection.py -s
+
+run_test_server_admin_suite: &run_test_server_admin_suite
+  run:
+    name: Runs the test_server_admin suite
+    no_output_timeout: 15m
+    command: |
+      set -x
+      pytest test/test_server_admin.py -s
+
+run_unit_tests: &run_unit_tests
+  run:
+    name: Runs the test_unit suite
+    no_output_timeout: 15m
+    command: |
+      set -x
+      pytest test/test_unit.py -s
+
+run_utils_tests: &run_utils_tests
+  run:
+    name: Runs the test_utils suite
+    no_output_timeout: 15m
+    command: |
+      set -x
+      pytest test/test_utils.py -s
+
+# See here for supported versions: https://devguide.python.org/versions/
+# We should support everything that's not EOL.
+run_test_supported_python_versions: &run_test_supported_python_versions
+  run:
+    name: Runs the test_utils suite against all supported python versions. 
+    no_output_timeout: 15m
+    command: |
+      set -x
+      tox
+
+setup_pytest: &setup_pytest
+  run:
+    name: Set up local env
+    command: |
+      # This is only required because circle might not keep their machine executor completetly up to date.
+      # apt-get update is failing because of this, hence we add this workaround. We don't use any heroku cli at all in here.
+      # Main issue here https://discuss.circleci.com/t/heroku-gpg-issues-in-ubuntu-images/43834/3
+      sudo rm -rf /etc/apt/sources.list.d/heroku.list
+
+      sudo apt-get update -y
+      sudo apt-get install python3-pip -y
+      pip3 install -r test-requirements.txt -r requirements.txt
+
+
+pull_license: &pull_license
+  run:
+    name: Pull license
+    command: |
+      echo ${pystardog_license} | base64 --decode  > ~/project/dockerfiles/stardog-license-key.bin
+
 jobs:
-  format:
+  static_analysis:
     docker:
-      - image: python:3.9
+      - image: python:latest
     steps:
       - checkout
       - run:
-          name: Formatter
+          name: Install dependencies
           command: |
             pip install -r test-requirements.txt
+
+      - run:
+          name: Black
+          command: |
             black --check .
 
-  single_node_tests:
-    docker:
-      - image: cimg/base:2020.01
+      - run:
+          # We fail hard if flake8 is not met.
+          name: Flake8
+          command: |
+            flake8 stardog 
+
+      - run:
+          # We allow pylint to fail since it's too strict.
+          # We should fix these slowly, and come up with a proper configuration to enable limited checks.
+          name: pylint
+          command: |
+            pylint stardog --exit-zero
+
+  basic_test_suite_single_node:
+    machine:
+      image: ubuntu-2204:2022.10.2
 
     steps:
       - checkout
 
+      - <<: *pull_license
+
       - run:
-          name: Pull license
+          name: Brings the single node and the required VGs up
+          no_output_timeout: 15m
           command: |
-            echo ${pystardog_license} | base64 --decode  > ~/project/dockerfiles/stardog-license-key.bin
+            set -x
+            docker-compose -f docker-compose.single-node.yml up -d
 
-      - setup_remote_docker
+      - run:
+          name: Waits for stardog stack to come up
+          command: |
+            source test/utils/wait.sh
+            wait_for_start_single_node localhost 5820
+
+      - <<: *setup_pytest
+      - <<: *run_basic_admin_suite
+      - <<: *run_connection_suite
+      - <<: *run_test_server_admin_suite
+      - <<: *run_utils_tests
+      - <<: *run_unit_tests
 
       - run:
-          name: Start containers and run single node test that are not compatible with cluster
+          name: Runs the single_node_only_test (These tests only work on single node Stardog instances)
           no_output_timeout: 15m
           command: |
             set -x
-            docker-compose -f docker-compose.single-node.yml up --exit-code-from tests-single-node
-
+            pytest test/test_single_node.py -s
+            
+      - <<: *installs_supported_python_versions
+      - <<: *run_test_supported_python_versions
+      - run:
+          name: Runs the single_node_only_test against multiple python envs
+          no_output_timeout: 15m
+          command: |
+            set -x
+            tox -e single_node
 
-  cluster_tests:
-    docker:
-      - image: cimg/base:2020.01
+  basic_test_cluster_mode:
+    machine:
+      image: ubuntu-2204:2022.10.2
 
     steps:
       - checkout
 
+      - <<: *pull_license
+
       - run:
-          name: Pull license
+          name: Start containers and cluster tests
+          no_output_timeout: 15m
           command: |
-            echo ${pystardog_license} | base64 --decode  > ~/project/dockerfiles/stardog-license-key.bin
+            set -x
+            docker-compose -f docker-compose.cluster.yml up -d
 
-      - setup_remote_docker
+      - <<: *setup_pytest
 
       - run:
-          name: Start containers and cluster tests
+          name: Waits for stardog stack to come up
+          command: |
+            source test/utils/wait.sh
+            wait_for_start_cluster localhost 5820
+
+      - <<: *run_basic_admin_suite
+      - <<: *run_connection_suite
+      - <<: *run_test_server_admin_suite
+      - <<: *run_utils_tests
+      - <<: *run_unit_tests
+
+      - run:
+          name: Runs the test the cluster only suite
+          no_output_timeout: 15m
+          command: |
+            set -x
+            pytest test/test_cluster.py -s
+
+      - <<: *installs_supported_python_versions
+      - run:
+          name: Runs the cluster node only tests against multiple python envs
           no_output_timeout: 15m
           command: |
             set -x
-            docker-compose -f docker-compose.cluster.yml up --exit-code-from tests
+            tox -e cluster
 
 workflows:
-  build_and_run_cluster_tests:
+  build_and_test:
     jobs:
-      - format
-      - single_node_tests
-      - cluster_tests
+      - static_analysis
+      - basic_test_suite_single_node
+      - basic_test_cluster_mode
+      # Have to reenable cache and standby tests
+      # - cache_tests
+      # - standby_tests
diff --git a/.gitignore b/.gitignore
@@ -105,6 +105,7 @@ venv.bak/
 # macos stuff
 .idea/
 .DS_Store
+.vscode
 
 # sd license files
 stardog-license-key.bin
diff --git a/README.md b/README.md
@@ -33,12 +33,30 @@ Docs](http://pystardog.readthedocs.io) or can be built using Sphinx:
 To run the tests locally, a valid Stardog license is required and placed in the `dockerfiles/stardog-license-key.bin`. 
 Docker and docker-compose are also required.
 
+1) Bring a stardog instance using docker-compose. For testing about 90% of the  pystardog features, just a single node is sufficient,
+although we also provide a cluster set up for further testing. 
 ```shell script
- docker-compose -f docker-compose.single-node.yml up --exit-code-from tests-single-node
- docker-compose -f docker-compose.cluster.yml up --exit-code-from tests
+# Bring a single node instance plus a bunch of Virtual Graphs for testing (Recommended).
+docker-compose -f docker-compose.single-node.yml up -d
 
+# A cluster set up is also provided, if cluster only features are to be implemented and tested.
+docker-compose -f docker-compose.cluster.yml up -d
 ```
 
+Run the test suite. Create a virtual environment with the neccesary dependencies:
+```shell script
+# Create a virtualenv and activate it
+virtualenv $(which python3) venv
+source venv/bin/activate
+
+# Install the dependencies
+pip install -r requirements.txt -r test-requirements.txt 
+
+# Run the basic test suite (covers most of the pystardog functionalities)
+pytest test/test_admin_basic.py test/test_connection.py test/test_utils.py -s 
+```
+
+
 ## Format
 To run a format of all the files
 ```shell script

diff --git a/docker-compose.cluster.yml b/docker-compose.cluster.yml
@@ -1,8 +1,4 @@
-# Note that circleci docker executor does not allow to volume mounting, hence the volume block can't be used here.
-# https://support.circleci.com/hc/en-us/articles/360007324514-How-can-I-use-Docker-volume-mounting-on-CircleCI-
-# For this reason, we are creating specific dockerfiles, and building the images from them, copying the files in the images
-# instead of mounting them.
-
+---
 version: "3.3"
 services:
   zoo1:
@@ -81,30 +77,13 @@ services:
       args:
         - TAG=${HAPROXY_TAG}
     container_name: ${STARDOG_LB}
+    ports:
+      - "127.0.0.1:5820:5820"
     depends_on:
       - zoo1
       - sd1
       - sd2
 
-  tests:
-    build:
-      context: .
-      dockerfile: dockerfiles/dockerfile-python
-    entrypoint: /bin/bash -c
-    command: ["./utils/run_tests.sh"]
-    container_name: pystardog_tests
-    environment:
-      - SSH_USER=${SSH_USER}
-      - SSH_PASS=${SSH_PASS}
-      - STARDOG_HOSTNAME_NODE_1=${STARDOG_HOSTNAME_NODE_1}
-      - STARDOG_HOSTNAME_CACHE=${STARDOG_HOSTNAME_CACHE}
-      - STARDOG_HOSTNAME_STANDBY=${STARDOG_HOSTNAME_STANDBY}
-      - STARDOG_ENDPOINT=http://${STARDOG_LB}:5820
-      - STARDOG_LB=${STARDOG_LB}
-      - STARDOG_USER=${STARDOG_USER}
-      - STARDOG_PASS=${STARDOG_PASS}
-
-
   # two mysql servers are used instead of one so we can simulate multiple datasources.
   mysql-music:
     build:

diff --git a/docker-compose.single-node.yml b/docker-compose.single-node.yml
@@ -1,12 +1,6 @@
-# Note that circleci docker executor does not allow to volume mounting, hence the volume block can't be used here.
-# https://support.circleci.com/hc/en-us/articles/360007324514-How-can-I-use-Docker-volume-mounting-on-CircleCI-
-# For this reason, we are creating specific dockerfiles, and building the images from them, copying the files in the images
-# instead of mounting them.
-
+---
 version: "3.3"
 services:
-
-  # So another node that does not depends on zookeper.
   sd-single-node:
     build:
       context: dockerfiles/
@@ -17,15 +11,20 @@ services:
     container_name: ${STARDOG_HOSTNAME_SINGLE_NODE}
     entrypoint: ["/bin/bash", "-c"]
     command: ["/var/start.sh"]
+    ports:
+      - "127.0.0.1:5820:5820"
+
+  # two mysql servers are used instead of one so we can simulate multiple datasources.
+  mysql-music:
+    build:
+      context: .
+      dockerfile: dockerfiles/dockerfile-mysql-music
+    # this value is hardcoded in conftest.py (music_options fixture), for multiple datasource / vg tests.
+    # it's also hardcoded in test/test_admin.py for testing imports.
+    container_name: pystardog_mysql_music
 
-  tests-single-node:
+  mysql-videos:
     build:
       context: .
-      dockerfile: dockerfiles/dockerfile-python
-    entrypoint: /bin/bash -c
-    command: ["./utils/run_test_single_node.sh"]
-    container_name: pystardog_tests_single_node
-    environment:
-      - STARDOG_ENDPOINT=${STARDOG_HOSTNAME_SINGLE_NODE}
-      - STARDOG_USER=${STARDOG_USER}
-      - STARDOG_PASS=${STARDOG_PASS}
+      dockerfile: dockerfiles/dockerfile-mysql-videos
+    container_name: pystardog_mysql_videos
diff --git a/dockerfiles/dockerfile-python b/dockerfiles/dockerfile-python
diff --git a/dockerfiles/dockerfile-stardog b/dockerfiles/dockerfile-stardog
@@ -16,6 +16,7 @@ COPY --from=base /tmp/mysql-connector-java.jar opt/stardog/server/dbms
 COPY stardog-license-key.bin /var/opt/stardog
 COPY start.sh /var/start.sh
 COPY start-standby.sh /var/start-standby.sh
+COPY example.ttl /tmp/example-remote.ttl
 
 USER root
 RUN yes $SSH_PASS | passwd $SSH_USER

diff --git a/dockerfiles/example.ttl b/dockerfiles/example.ttl
@@ -0,0 +1 @@
+<urn:subj> <urn:pred> <urn:obj> .
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
-requests==2.22.0
+requests==2.28.1
 requests-toolbelt==0.9.1
 contextlib2==0.5.5
 recommonmark==0.5.0