From 09d13e1eafcc186dc3d3e658987da0e8c0684705 Mon Sep 17 00:00:00 2001 From: Levi <141682181+levisingularity@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:17:45 -0300 Subject: [PATCH] Setup atomdb and query engine from the host machine (#61) * make local run * feat: setup pythonpath from host machine * Set up environment variable ATOMDB_PACKAGE_PATH and QUERY_ENGINE_PACKAGE_PATH to bind the packages from the host machine into the container * docs: update readme * Add a section explaining how to setup the variables ATOMDB_PACKAGE_PATH and QUERY_ENGINE_PACKAGE_PATH * Update the commands to start and stop the function * Update commands to run tests * fix: set missing environment variables --- .dockerignore | 7 +++++++ .env.example | 4 +++- Makefile | 8 +++++++- README.md | 34 +++++++++++++++++++++++++++++----- docker-compose.yaml | 14 +++++++++++++- scripts/initd.sh | 18 +++++++++++++++++- 6 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f81c736 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +env/ +.vscode +docs +build +.github +__pycache__ +.pytest_cache diff --git a/.env.example b/.env.example index 2900ca5..22064a7 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,8 @@ DAS_MONGODB_USERNAME=${MONGO_INITDB_ROOT_USERNAME} DAS_MONGODB_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD} DAS_DATABASE_USERNAME=${MONGO_INITDB_ROOT_USERNAME} DAS_DATABASE_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD} -PYTHONPATH=/app + +ATOMDB_PACKAGE_PATH= +QUERY_ENGINE_PACKAGE_PATH= DAS_KNOWLEDGE_BASE=/data/samples/animals.metta diff --git a/Makefile b/Makefile index 0130ce4..a10d52d 100644 --- a/Makefile +++ b/Makefile @@ -18,4 +18,10 @@ unit-tests-coverage: integration-tests: ./scripts/run-tests.sh integration -pre-commit: unit-tests-coverage lint \ No newline at end of file +pre-commit: unit-tests-coverage lint + +serve: + @docker compose up --build --force-recreate -d + +stop: + @docker compose down diff --git a/README.md b/README.md index 18acb40..355a121 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ OpenFaaS, or Open Functions as a Service, is an open-source platform simplifying ## Running an OpenFaaS Function Locally ### Requirements + - Docker - Docker Compose @@ -33,11 +34,10 @@ The project architecture consists of: - **das-query-engine**: Operates on the same network as the 'openfaas' container. - **Port 8080**: Exposed for connection to the host machine. - ### Step-by-Step Guide 1. **Cloning the Project** - + Clone the project repository to your local environment. ```bash @@ -58,7 +58,7 @@ The project architecture consists of: Execute the following command at the root of the project to start the required services: ```bash - docker compose up -d + make serve ``` This will start containers for Redis, MongoDB, and a temporary container named 'canonical-load'. The latter is used to load initial data into the Redis and MongoDB databases. After its execution, a container named 'openfaas' will start, which includes the faas-cli. @@ -72,7 +72,7 @@ The project architecture consists of: To shut down the containers and clean up the environment, execute the following command: ```bash - docker compose down + make stop ``` ### Testing @@ -80,12 +80,36 @@ The project architecture consists of: To run automated tests for the project, use the following script: ```bash -./scripts/run-tests.sh +make integration-tests +``` + +```bash +make unit-tests ``` This script will set up the necessary environment for testing. +### Use the hyperon-das and hyperon-das-atomdb from the host machine + +This feature has been implemented to allow developers to test the integration of AtomDB and Query Engine packages locally, even before publishing them on PyPI. This facilitates efficient testing during the development phase. +1. Open the `.env` file at the root of your project. + +2. Add the following environment variables, adjusting the paths as necessary: + + ```dotenv + ATOMDB_PACKAGE_PATH=/path/to/your/atomdb/package + QUERY_ENGINE_PACKAGE_PATH=/path/to/your/query/engine/package + ``` + + Make sure to replace `/path/to/your/atomdb/package` and `/path/to/your/query/engine/package` with the actual paths of the AtomDB and Query Engine packages on your system. + +3. If you prefer to use the latest versions of the AtomDB and Query Engine packages published on PyPI, leave the variables empty: + + ```dotenv + ATOMDB_PACKAGE_PATH= + QUERY_ENGINE_PACKAGE_PATH= + ``` ## Obtaining Function Logs in faasd Using the `ctr` Command diff --git a/docker-compose.yaml b/docker-compose.yaml index 7489dcb..88ae876 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -41,6 +41,8 @@ services: command: python3 scripts/load_das.py --knowledge-base ${DAS_KNOWLEDGE_BASE} volumes: - /tmp:/tmp + environment: + - PYTHONPATH=/app env_file: - .env depends_on: @@ -52,7 +54,17 @@ services: privileged: true build: . volumes: - - ./das-query-engine:/usr/local/function/das-query-engine + - type: bind + source: ./das-query-engine + target: /usr/local/function/das-query-engine + + - type: bind + source: ${ATOMDB_PACKAGE_PATH:-/tmp} + target: /opt/repos/hyperon_das_atomdb + + - type: bind + source: ${QUERY_ENGINE_PACKAGE_PATH:-/tmp} + target: /opt/repos/hyperon_das ports: - 8080:8080 env_file: diff --git a/scripts/initd.sh b/scripts/initd.sh index 9688573..7746283 100755 --- a/scripts/initd.sh +++ b/scripts/initd.sh @@ -13,5 +13,21 @@ else done fi -faas-cli local-run --network host --watch +faas-cli build + +FUNCTION_NAME=$(grep -A6 "^functions:" "./stack.yml" | grep -E "^ +query-engine:" -A8 | awk '/^ +image:/ {print $2}') + +docker run --rm --name query-engine \ + --network host \ + -v /opt/repos:/opt/repos \ + -e PYTHONPATH=/opt/repos \ + -e DAS_MONGODB_NAME \ + -e DAS_MONGODB_HOSTNAME \ + -e DAS_MONGODB_PORT \ + -e DAS_REDIS_HOSTNAME \ + -e DAS_REDIS_PORT \ + -e DAS_MONGODB_USERNAME \ + -e DAS_MONGODB_PASSWORD \ + $FUNCTION_NAME + #tail -f /dev/null