Dev

.github/workflows/test_app.yml

@@ -3,16 +3,52 @@
 name: Test Source Collector App
 on: pull_request
 
+#jobs:
+#    build:
+#        runs-on: ubuntu-latest
+#        steps:
+#            - name: Checkout repository
+#              uses: actions/checkout@v4
+#            - name: Run docker-compose
+#              uses: hoverkraft-tech/[email protected]
+#              with:
+#                  compose-file: "docker-compose.yml"
+#            - name: Execute tests in the running service
+#              run: |
+#                  docker ps -a && docker exec data-source-identification-app-1 pytest /app/tests/test_automated
+
 jobs:
-    build:
-        runs-on: ubuntu-latest
-        steps:
-            - name: Checkout repository
-              uses: actions/checkout@v4
-            - name: Run docker-compose
-              uses: hoverkraft-tech/[email protected]
-              with:
-                  compose-file: "docker-compose.yml"
-            - name: Execute tests in the running service
-              run: |
-                  docker exec data-source-identification-app-1 pytest /app/tests/test_automated
+  container-job:
+    runs-on: ubuntu-latest
+    container: python:3.12.8
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Run tests
+        run: |
+          pytest tests/test_automated
+          pytest tests/test_alembic
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_DB: postgres
+          POSTGRES_HOST: postgres
+          POSTGRES_PORT: 5432
+          GOOGLE_API_KEY: TEST
+          GOOGLE_CSE_ID: TEST

Dockerfile

@@ -14,5 +14,4 @@ RUN pip install --no-cache-dir -r requirements.txt
 # Expose the application port
 EXPOSE 80
 
-# Run FastAPI app with uvicorn
-CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "80"]
+RUN chmod +x execute.sh

ENV.md

@@ -15,3 +15,4 @@ Please ensure these are properly defined in a `.env` file in the root directory.
 |`POSTGRES_HOST`                 | The host for the test database                                                                                                                                                                                                                                    | `127.0.0.1` |
 |`POSTGRES_PORT`                 | The port for the test database                                                                                                                                                                                                                                    | `5432` |
 |`DS_APP_SECRET_KEY`| The secret key used for decoding JWT tokens produced by the Data Sources App. Must match the secret token that is used in the Data Sources App for encoding.                                                                                                      |`abc123`|
+|`DEV`| Set to any value to run the application in development mode.                                                                                                                                                                                                      |`true`|

README.md

@@ -45,6 +45,12 @@ This can be done via the following command:
 docker compose up -d
 ```
 
+Following that, you will need to set up the uvicorn server using the following command:
+
+```bash
+docker exec data-source-identification-app-1 uvicorn api.main:app --host 0.0.0.0 --port 80
+```
+
 Note that while the container may mention the web app running on `0.0.0.0:8000`, the actual host may be `127.0.0.1:8000`.
 
 To access the API documentation, visit `http://{host}:8000/docs`.

alembic.ini

@@ -0,0 +1,117 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+# Use forward slashes (/) also on windows to provide an os agnostic path
+script_location = collector_db/alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+# version_path_separator = newline
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = postgresql://test_source_collector_user:[email protected]:5432/source_collector_test_db
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

api/main.py

@@ -9,12 +9,14 @@
 from collector_db.DatabaseClient import DatabaseClient
 from core.CoreLogger import CoreLogger
 from core.SourceCollectorCore import SourceCollectorCore
+from util.helper_functions import get_from_env
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     # Initialize shared dependencies
     db_client = DatabaseClient()
+    await setup_database(db_client)
     source_collector_core = SourceCollectorCore(
         core_logger=CoreLogger(
             db_client=db_client
@@ -34,6 +36,15 @@
     pass
 
 
+async def setup_database(db_client):
+    # Initialize database if dev environment, otherwise apply migrations
+    try:
+        get_from_env("DEV")
+        db_client.init_db()
+    except Exception as e:
+        return
+
+
 app = FastAPI(
     title="Source Collector API",
     description="API for collecting data sources",

apply_migrations.py

@@ -0,0 +1,14 @@
+from alembic import command
+from alembic.config import Config
+
+from collector_db.helper_functions import get_postgres_connection_string
+
+if __name__ == "__main__":
+    print("Applying migrations...")
+    alembic_config = Config("alembic.ini")
+    alembic_config.set_main_option(
+        "sqlalchemy.url",
+        get_postgres_connection_string()
+    )
+    command.upgrade(alembic_config, "head")
+    print("Migrations applied.")

collector_db/DTOs/URLInfo.py

@@ -1,3 +1,4 @@
+import datetime
 from typing import Optional
 
 from pydantic import BaseModel
@@ -11,3 +12,4 @@
     url: str
     url_metadata: Optional[dict] = None
     outcome: URLOutcome = URLOutcome.PENDING
+    updated_at: Optional[datetime.datetime] = None

collector_db/DatabaseClient.py

@@ -16,7 +16,6 @@
 from collector_db.helper_functions import get_postgres_connection_string
 from collector_db.models import Base, Batch, URL, Log, Duplicate
 from collector_manager.enums import CollectorType
-from core.DTOs.BatchStatusInfo import BatchStatusInfo
 from core.enums import BatchStatus
 
 
@@ -32,10 +31,12 @@
             url=db_url,
             echo=ConfigManager.get_sqlalchemy_echo(),
         )
-        Base.metadata.create_all(self.engine)
         self.session_maker = scoped_session(sessionmaker(bind=self.engine))
         self.session = None
 
+    def init_db(self):
+        Base.metadata.create_all(self.engine)
+
     def session_manager(method):
         @wraps(method)
         def wrapper(self, *args, **kwargs):
@@ -214,13 +215,13 @@
         # Get only the batch_id, collector_type, status, and created_at
         limit = 100
         query = (session.query(Batch)
-                 .order_by(Batch.date_generated.desc())
-                 .limit(limit)
-                 .offset((page - 1) * limit))
+                 .order_by(Batch.date_generated.desc()))
         if collector_type:
             query = query.filter(Batch.strategy == collector_type.value)
         if status:
             query = query.filter(Batch.status == status.value)
+        query = (query.limit(limit)
+                 .offset((page - 1) * limit))
         batches = query.all()
         return [BatchInfo(**batch.__dict__) for batch in batches]
 
@@ -274,6 +275,11 @@
             Log.created_at < datetime.now() - timedelta(days=1)
         ).delete()
 
+    @session_manager
+    def update_url(self, session, url_info: URLInfo):
+        url = session.query(URL).filter_by(id=url_info.id).first()
+        url.url_metadata = url_info.url_metadata
+
 if __name__ == "__main__":
     client = DatabaseClient()
     print("Database client initialized.")

collector_db/alembic/README.md

@@ -1,4 +1,30 @@
-Generic single-database configuration.
+Alembic is a lightweight Python library that helps manage database migrations.
 
+## Files and Directories
+
+The following files are present in this directory OR related to it:
+- `script.py.mako`: This is a Mako template file which is used to generate new migration scripts. Whatever is here is used to generate new files within `versions/`. This is scriptable so that the structure of each migration file can be controlled, including standard imports to be within each, as well as changes to the structure of the `upgrade()` and `downgrade()` functions
+- `env.py`: The main script that sets up the migration environment.
+- `alembic.ini`: The `alembic` configuration file. Located in the root of the repository
+- `/versions`: The directory which contains the migration scripts
+- `apply_migrations.py`: A Python script, located in the root directory, which applies any outstanding migrations to the database
+- `execute.sh`: A shell script in the root directory which runs the `apply_migrations.py` script. Called by DigitalOcean when deploying the application.
+
+## Generating a Migration
+
+To generate a new migration, run the following command from the root directory:
+
+```bash
+alembic revision --autogenerate -m "Description for migration"
+```
+
+Then, locate the new revision script in `/versions` and modify the update and downgrade functions as needed
+
+Once you have generated a new migration, you can upgrade and downgrade the database using the `alembic` command line tool.
+
+Finally, make sure to commit your changes to the repository.
+
+## How does Alembic Work?
+
+As long as new migrations are generated and stored in the `/versions` directory, Alembic will apply them, in the order they were made, to the production database.
 
-- `script.py.mako`: This is a Mako template file which is used to generate new migration scripts. Whatever is here is used to generate new files within `versions/`. This is scriptable so that the structure of each migration file can be controlled, including standard imports to be within each, as well as changes to the structure of the `upgrade()` and `downgrade()` functions

collector_db/alembic/env.py

@@ -5,6 +5,9 @@
 
 from alembic import context
 
+from collector_db.helper_functions import get_postgres_connection_string
+from collector_db.models import Base
+
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
@@ -18,7 +21,7 @@
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-target_metadata = None
+target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -38,7 +41,7 @@ def run_migrations_offline() -> None:
     script output.
 
     """
-    url = config.get_main_option("sqlalchemy.url")
+    url = get_postgres_connection_string()
     context.configure(
         url=url,
         target_metadata=target_metadata,