CI to detect out of sync migrations (#635)

- test for a change to migrations
- actually testing with base reqs
- skip assistant tests if not installed
- upload requirements in dev

.github/workflows/publish-test.yml

@@ -31,7 +31,7 @@ jobs:
     - uses: actions/setup-node@v4
       with:
         node-version-file: '.nvmrc'
-    - name: Install dependencies
+    - name: Install npm dependencies
       run: npm install
     - name: Build client
       run: npm run build

.github/workflows/test.yml

@@ -85,7 +85,6 @@ jobs:
           python -m coverage report
 
       - name: Upload HTML report
-        if: ${{ failure() }}
         uses: actions/upload-artifact@v4
         with:
           name: html-report

explorer/ee/db_connections/utils.py

@@ -2,8 +2,6 @@
 from django.db.utils import load_backend
 import os
 
-from dateutil import parser
-
 import sqlite3
 import io
 
@@ -126,6 +124,7 @@ def atof_custom(value):
 
 def csv_to_typed_df(csv_bytes, delimiter=",", has_headers=True):  # noqa
     import pandas as pd
+    from dateutil import parser
     try:
 
         csv_file = io.BytesIO(csv_bytes)

explorer/tests/settings.py

@@ -1,7 +1,7 @@
 from test_project.settings import *  # noqa
 
 EXPLORER_ENABLE_ANONYMOUS_STATS = False
-EXPLORER_TASKS_ENABLED = False  # set to true to test async tasks
+EXPLORER_TASKS_ENABLED = True  # set to true to test async tasks
 EXPLORER_AI_API_KEY = None  # set to any value to enable assistant
 CELERY_BROKER_URL = "redis://localhost:6379/0"
 CELERY_TASK_ALWAYS_EAGER = True

explorer/tests/settings_base.py

@@ -0,0 +1,4 @@
+from explorer.tests.settings import *  # noqa
+
+EXPLORER_TASKS_ENABLED = False
+EXPLORER_USER_UPLOADS_ENABLED = False

explorer/tests/test_apps.py

@@ -1,8 +1,9 @@
 from unittest.mock import patch
+from io import StringIO
 
 from django.core.exceptions import ImproperlyConfigured
 from django.test import TestCase
-
+from django.core.management import call_command
 from explorer.apps import _validate_connections
 
 
@@ -17,3 +18,18 @@ def test_validates_default_connections(self, mocked_connection):
     def test_validates_all_connections(self, mocked_connections):
         mocked_connections.return_value = {"garbage1": "in", "garbage2": "out"}
         self.assertRaises(ImproperlyConfigured, _validate_connections)
+
+
+class PendingMigrationsTests(TestCase):
+
+    def test_no_pending_migrations(self):
+        out = StringIO()
+        try:
+            call_command(
+                "makemigrations",
+                "--check",
+                stdout=out,
+                stderr=StringIO(),
+            )
+        except SystemExit:  # noqa
+            self.fail("Pending migrations:\n" + out.getvalue())

explorer/tests/test_assistant.py

@@ -1,5 +1,7 @@
 from explorer.tests.factories import SimpleQueryFactory, QueryLogFactory
 from unittest.mock import patch, Mock, MagicMock
+import unittest
+from explorer import app_settings
 
 import json
 from django.test import TestCase
@@ -10,6 +12,7 @@
 from explorer.assistant.utils import sample_rows_from_table, ROW_SAMPLE_SIZE, build_prompt
 
 
+@unittest.skipIf(not app_settings.has_assistant(), "assistant not enabled")
 class TestAssistantViews(TestCase):
 
     def setUp(self):
@@ -43,6 +46,7 @@ def test_assistant_help(self, mocked_openai_client):
         self.assertEqual(json.loads(resp.content)["message"], "smart computer")
 
 
+@unittest.skipIf(not app_settings.has_assistant(), "assistant not enabled")
 class TestBuildPrompt(TestCase):
 
     @patch("explorer.assistant.utils.sample_rows_from_tables", return_value="sample data")
@@ -145,6 +149,7 @@ def test_build_prompt_with_extra_tables_not_fitting_window(self, mock_get_item,
         self.assertEqual(result["system"], "system prompt")
 
 
+@unittest.skipIf(not app_settings.has_assistant(), "assistant not enabled")
 class TestPromptContext(TestCase):
 
     def test_retrieves_sample_rows(self):
@@ -263,6 +268,7 @@ def test_schema_info_from_table_names(self):
         self.assertEqual(ret, expected)
 
 
+@unittest.skipIf(not app_settings.has_assistant(), "assistant not enabled")
 class TestAssistantUtils(TestCase):
 
     def test_sample_rows_from_tables(self):

requirements/dev.txt

@@ -3,7 +3,9 @@
 -r ./extra/charts.txt
 -r ./extra/snapshots.txt
 -r ./extra/xls.txt
+-r ./extra/uploads.txt
+-r ./tests.txt
 
-importlib-metadata<5.0; python_version <= '3.7'
-coverage
-factory-boy>=3.1.0
+# The Celery broker that test_project uses. Not required if not using async tasks, or if you have
+# a Celery config that uses a different broker.
+redis>=5.0

requirements/extra/snapshots.txt

@@ -1,2 +1,2 @@
-boto3>=1.3
+boto3>=1.30.0
 celery>=4.0

requirements/tests.txt

@@ -0,0 +1,5 @@
+-r ./base.txt
+
+importlib-metadata<5.0; python_version <= '3.7'
+coverage
+factory-boy>=3.1.0

tox.ini

@@ -10,15 +10,16 @@ skip_missing_interpreters=True
 [testenv]
 allowlist_externals = coverage
 deps =
-    base-reqs: -r requirements/dev.txt
+    base-reqs: -r requirements/tests.txt
     dj32: django>=3.2,<4.0
     dj42: django>=4.2,<5.0
     dj50: django>=5.0,<5.1
     djmain: https://github.com/django/django/archive/main.tar.gz
     dev: -r requirements/dev.txt
 commands =
     {envpython} --version
-    {env:COMMAND:coverage} run manage.py test --settings=explorer.tests.settings --noinput
+    base-reqs: coverage run manage.py test --settings=explorer.tests.settings_base --noinput
+    dev: coverage run manage.py test --settings=explorer.tests.settings --noinput
 ignore_outcome =
     djmain: True
 ignore_errors =