Merge pull request #1 from JSv4/JSv4/cleanup-and-add-tests

Cleanup

.dockerignore

@@ -1,4 +1,5 @@
-frontend
+.git
+.pytest_cache
 .editorconfig
 .gitattributes
 .github
@@ -8,5 +9,7 @@ frontend
 .pre-commit-config.yaml
 .readthedocs.yml
 .travis.yml
+delphic/media/*
 venv
-.git
+docs
+frontend

.envs/.test/.django

@@ -0,0 +1,25 @@
+# General
+# ------------------------------------------------------------------------------
+USE_DOCKER=yes
+IPYTHONDIR=/app/.ipython
+
+# Redis
+# ------------------------------------------------------------------------------
+REDIS_URL=redis://redis:6379/0
+
+# Celery
+# ------------------------------------------------------------------------------
+
+# Flower
+CELERY_FLOWER_USER=AbFgrzUdIweDSBYEiGuhipavkoTonmNo
+CELERY_FLOWER_PASSWORD=XdBJp2EnHZ5xQc2r3FSYjmTGbHiJp3wf5yWJnKloHwsrSLC3qfdoF4naZfwuZYBU
+
+# DJANGO SETTINGS
+# -----------------------------------------------------------------------------
+DJANGO_SETTINGS_MODULE=config.settings.test
+
+# NLP / LLM
+# -----------------------------------------------------------------------------
+OPENAI_API_KEY="1234"
+MODEL_NAME=text-davinci-003
+MAX_TOKENS=2048

.envs/.test/.postgres

@@ -0,0 +1,7 @@
+# PostgreSQL
+# ------------------------------------------------------------------------------
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432
+POSTGRES_DB=cookietest
+POSTGRES_USER=bbQXlhtKFPTMHhhCaplnwWQMFFpkTjpP
+POSTGRES_PASSWORD=ooEjyQ1uDy4W7KMwZeXnLOQNdkgCuX02STBJv4YxP1o0siqCWWW4zVOrj49KWV96

.github/dependabot.yml

@@ -0,0 +1,91 @@
+# Config for Dependabot updates. See Documentation here:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  # Update GitHub actions in workflows
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    # Every weekday
+    schedule:
+      interval: 'daily'
+
+  # Enable version updates for Docker
+  # We need to specify each Dockerfile in a separate entry because Dependabot doesn't
+  # support wildcards or recursively checking subdirectories. Check this issue for updates:
+  # https://github.com/dependabot/dependabot-core/issues/2178
+  - package-ecosystem: 'docker'
+    # Look for a `Dockerfile` in the `compose/local/django` directory
+    directory: 'compose/local/django/'
+    # Every weekday
+    schedule:
+      interval: 'daily'
+    # Ignore minor version updates (3.10 -> 3.11) but update patch versions
+    ignore:
+      - dependency-name: '*'
+        update-types:
+          - 'version-update:semver-major'
+          - 'version-update:semver-minor'
+
+  - package-ecosystem: 'docker'
+    # Look for a `Dockerfile` in the `compose/local/docs` directory
+    directory: 'compose/local/docs/'
+    # Every weekday
+    schedule:
+      interval: 'daily'
+    # Ignore minor version updates (3.10 -> 3.11) but update patch versions
+    ignore:
+      - dependency-name: '*'
+        update-types:
+          - 'version-update:semver-major'
+          - 'version-update:semver-minor'
+
+  - package-ecosystem: 'docker'
+    # Look for a `Dockerfile` in the `compose/local/node` directory
+    directory: 'compose/local/node/'
+    # Every weekday
+    schedule:
+      interval: 'daily'
+
+  - package-ecosystem: 'docker'
+    # Look for a `Dockerfile` in the `compose/production/aws` directory
+    directory: 'compose/production/aws/'
+    # Every weekday
+    schedule:
+      interval: 'daily'
+
+  - package-ecosystem: 'docker'
+    # Look for a `Dockerfile` in the `compose/production/django` directory
+    directory: 'compose/production/django/'
+    # Every weekday
+    schedule:
+      interval: 'daily'
+    # Ignore minor version updates (3.10 -> 3.11) but update patch versions
+    ignore:
+      - dependency-name: '*'
+        update-types:
+          - 'version-update:semver-major'
+          - 'version-update:semver-minor'
+
+  - package-ecosystem: 'docker'
+    # Look for a `Dockerfile` in the `compose/production/postgres` directory
+    directory: 'compose/production/postgres/'
+    # Every weekday
+    schedule:
+      interval: 'daily'
+
+  - package-ecosystem: 'docker'
+    # Look for a `Dockerfile` in the `compose/production/traefik` directory
+    directory: 'compose/production/traefik/'
+    # Every weekday
+    schedule:
+      interval: 'daily'
+
+  # Enable version updates for Python/Pip - Production
+  - package-ecosystem: 'pip'
+    # Look for a `requirements.txt` in the `root` directory
+    # also 'setup.cfg', 'runtime.txt' and 'requirements/*.txt'
+    directory: '/'
+    # Every weekday
+    schedule:
+      interval: 'daily'

.github/workflows/ci.yml

@@ -0,0 +1,54 @@
+name: CI
+
+# Enable Buildkit and let compose use it to speed up image building
+env:
+  DOCKER_BUILDKIT: 1
+  COMPOSE_DOCKER_CLI_BUILD: 1
+
+on:
+  pull_request:
+    branches: ['master', 'main']
+    paths-ignore: ['docs/**']
+
+  push:
+    branches: ['master', 'main']
+    paths-ignore: ['docs/**']
+
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  linter:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code Repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      # Consider using pre-commit.ci for open source project
+      - name: Run pre-commit
+        uses: pre-commit/[email protected]
+
+  # With no caching at all the entire ci process takes 4m 30s to complete!
+  pytest:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code Repository
+        uses: actions/checkout@v3
+
+      - name: Build the Stack
+        run: docker-compose -f test.yml build
+
+      - name: Run DB Migrations
+        run: docker-compose -f test.yml run --rm django python manage.py migrate
+
+      - name: Run Django Tests
+        run: docker-compose -f test.yml run django pytest
+
+      - name: Tear down the Stack
+        run: docker-compose -f test.yml down

.gitignore

@@ -331,9 +331,10 @@ chat_all_the_docs/media/
 
 .pytest_cache/
 .ipython/
-.envs/*
+.envs/.local
+.envs/.production
 !.envs/.local/
-
+delphic/media/*
 
 ### Models for Question Answering
 cache/*

README.md

@@ -1,8 +1,8 @@
 # Delphic
 
-![](./docs/images/Delphic.png)
+![](./docs/images/Delphic Header.png)
 
-A simple framework to build and deploy LLM agents that can be used to analyze and manipulate text data from documents. 
+A simple framework to use LlamaIndex to build and deploy LLM agents that can be used to analyze and manipulate text data from documents.
 
 [![Built with Cookiecutter Django](https://img.shields.io/badge/built%20with-Cookiecutter%20Django-ff69b4.svg?logo=cookiecutter)](https://github.com/cookiecutter/cookiecutter-django/)
 [![Black code style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black)
@@ -19,9 +19,9 @@ The initial release of Delphic is based *solely* on OpenAI's API. We fully plan
 
 #### Just Run the Application
 
-The fastest way to get up and running is to clone this repo and then deploy the application locally. 
+The fastest way to get up and running is to clone this repo and then deploy the application locally.
 
-You will need Docker and Docker Compose to follow these instructions. DigitalOcean, besides being an excellent cloud host, has some of the easiest-to-follow instructions on setting these up. Please check them out here or go to the Docker official instructions. 
+You will need Docker and Docker Compose to follow these instructions. DigitalOcean, besides being an excellent cloud host, has some of the easiest-to-follow instructions on setting these up. Please check them out here or go to the Docker official instructions.
 
 1. First, clone the repo:
 
@@ -57,14 +57,14 @@ sudo docker-compose --profile fullstack -f local.yml build
 6. Finally, to launch the application, type:
 
 ```commandline
-sudo docker-compose --profile fullstack -f local.yml up 
+sudo docker-compose --profile fullstack -f local.yml up
 ```
 
 Go to `localhost:3000` to see the frontend.
 
 #### I Want to Develop / Modify the Frontend
 
-*If you want to actively develop the frontend, we suggest you **NOT** use the `--profile=fullstack` flag as every change will require a full container rebuild. Instead, instead of step #5 above, 
+*If you want to actively develop the frontend, we suggest you **NOT** use the `--profile=fullstack` flag as every change will require a full container rebuild. Instead, instead of step #5 above,
 
 5. First, launch the backend without the fullstack flag:
 
@@ -75,7 +75,7 @@ sudo docker-compose -f local.yml up
 6. In one terminal window, launch your backend:
 
 ```commandline
-sudo docker-compose -f local.yml up 
+sudo docker-compose -f local.yml up
 ```
 
 7. Now, in a separate terminal, cd into the frontend directory and start a development server (**Note, we assume you have nvm installed. If you don't install it now**):
@@ -92,18 +92,52 @@ Go to `localhost:3000` to see the frontend.
 
 ### Production Deploy
 
-This assumes you want to make the application available to the internet at some kind of fully qualified domain like delphic.opensource.legal. In order to do this, you need to update a couple configurations. 
+This assumes you want to make the application available to the internet at some kind of fully qualified domain like delphic.opensource.legal. In order to do this, you need to update a couple configurations.
 
 **TODO - insert documentation**
 
+### Development
+
+If you want to contribute to Delphic or roll your own version, you'll want to ensure you setup the development environment.
+
+#### Backend Setup
+
+On the backend, you'll need to have a working python environment to run the pre-commit formatting checks. You can use
+your system python interpreter, but we recommend using pyenv and creating a virtual env based off of Python>=3.10.
+
+##### Pre-Commit Setup
+
+Then, in the root of your local repo, run these commands:
+
+```
+pip install -r ./requirements/local.txt
+pre-commit install
+```
+
+Now, when you stage your commits, ou ar code formatting and style checks will run automatically.
+
+##### Running Tests
+
+We have a basic test suite in `./tests`. You can run the tests by typing:
+
+```commandline
+sudo docker-compose -f local.yml run django python manage.py test
+```
+
+#### Frontend Setup
+
+On the frontend, we're using node v18.15.0. We assume you're using nvm. We don't have any frontend tests yet (sorry).
+You'll use want to cd into the frontend directory (`cd frontend`) and type `nvm use` to activate the appropriate node
+version. Then you can type `yarn install` and finally `yarn run start` to get started.
+
 ## Using the Application
 
 ### Warning / Disclaimer
 
 At the moment, any user who is logged in will have full permissions. We plan to implement the more precise, roles-based access control module we developed for [OpenContracts](https://github.com/JSv4/OpenContracts), but, for now
-be aware that anyone with any type of login credentials can create and delete collections. **Creating collections uses OpenAI credits / costs money**. 
+be aware that anyone with any type of login credentials can create and delete collections. **Creating collections uses OpenAI credits / costs money**.
 
-If you want to create a super user, you can follow the typical django command (from the repo root in your local filesystem): 
+If you want to create a super user, you can follow the typical django command (from the repo root in your local filesystem):
 
 ```commandline
 sudo docker-compose -f local.yml run django python manage.py createsuperuser
@@ -121,7 +155,3 @@ https://user-images.githubusercontent.com/5049984/233236432-aa4980b6-a510-42f3-8
 ### Creating a Collection
 
 **TODO - add gif**
-
-
-
-