Skip to content

Commit

Permalink
Add trivial unit test for database instantiation and querying empty c…
Browse files Browse the repository at this point in the history
…hunks table.
  • Loading branch information
andy-slac committed Apr 16, 2024
1 parent af6e00a commit 03d4c2d
Show file tree
Hide file tree
Showing 9 changed files with 704 additions and 35 deletions.
67 changes: 67 additions & 0 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
name: build_and_test

on:
push:
branches:
- main
pull_request:

jobs:
build_and_test:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.11", "3.12"]

steps:
- uses: actions/checkout@v3
with:
# Need to clone everything for the git tags.
fetch-depth: 0

- uses: conda-incubator/setup-miniconda@v2
with:
python-version: ${{ matrix.python-version }}
channels: conda-forge,defaults
channel-priority: strict
show-channel-urls: true
miniforge-variant: Mambaforge
use-mamba: true

- name: Update pip/wheel infrastructure
shell: bash -l {0}
run: |
mamba install -y -q pip wheel
pip install uv
- name: Install dependencies
shell: bash -l {0}
run: |
uv pip install -r requirements.txt
# We have two cores so we can speed up the testing with xdist
- name: Install pytest packages
shell: bash -l {0}
run: |
uv pip install \
pytest pytest-xdist pytest-cov
- name: List installed packages
shell: bash -l {0}
run: |
conda list
pip list -v
- name: Build and install
shell: bash -l {0}
run: |
uv pip install -v --no-deps -e .
- name: Run tests
shell: bash -l {0}
run: |
pytest -r a -v -n 3 --cov=lsst.dax.ppdb --cov=tests --cov-report=xml --cov-report=term --cov-branch
- name: Upload coverage to codecov
uses: codecov/codecov-action@v2
with:
file: ./coverage.xml
24 changes: 24 additions & 0 deletions python/lsst/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# This file is part of dax_ppdb.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (https://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

import pkgutil

__path__ = pkgutil.extend_path(__path__, __name__)
24 changes: 24 additions & 0 deletions python/lsst/dax/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# This file is part of dax_ppdb.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (https://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

import pkgutil

__path__ = pkgutil.extend_path(__path__, __name__)
1 change: 1 addition & 0 deletions python/lsst/dax/ppdb/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

from .config import *
from .ppdb import *
from .version import * # Generated by sconsUtils
111 changes: 76 additions & 35 deletions python/lsst/dax/ppdb/sql/_ppdb_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,18 @@
from typing import Any

import astropy.time
import felis.datamodel
import sqlalchemy
import yaml
from felis.datamodel import Schema, SchemaVersion
from felis.metadata import MetaDataBuilder
from lsst.dax.apdb import ApdbMetadata, ApdbTableData, IncompatibleVersionError, ReplicaChunk, VersionTuple
from lsst.dax.apdb.sql.apdbMetadataSql import ApdbMetadataSql
from lsst.dax.apdb.sql.apdbSqlSchema import GUID
from lsst.dax.apdb import (
ApdbMetadata,
ApdbTableData,
IncompatibleVersionError,
ReplicaChunk,
VersionTuple,
schema_model,
)
from lsst.dax.apdb.sql import ApdbMetadataSql, ModelToSql
from lsst.resources import ResourcePath
from lsst.utils.iteration import chunk_iterable
from sqlalchemy import sql
Expand Down Expand Up @@ -127,13 +132,13 @@ def __init__(self, config: PpdbConfig):
def init_database(
cls,
db_url: str,
schema_name: str | None,
schema_file: str | None,
felis_schema: str | None,
use_connection_pool: bool,
isolation_level: str | None,
connection_timeout: float | None,
drop: bool,
schema_file: str | None = None,
schema_name: str | None = None,
felis_schema: str | None = None,
use_connection_pool: bool = True,
isolation_level: str | None = None,
connection_timeout: float | None = None,
drop: bool = False,
) -> PpdbConfig:
"""Initialize PPDB database.
Expand Down Expand Up @@ -216,40 +221,76 @@ def _read_schema(
table for table in schema_dict["tables"] if table["name"] not in ("DiaObjectLast",)
]
schema_dict["tables"] = filtered_tables
schema = Schema.model_validate(schema_dict)
dm_schema = felis.datamodel.Schema.model_validate(schema_dict)
schema = schema_model.Schema.from_felis(dm_schema)

# Replace schema name with a configured one, this helps in case we
# want to use default schema on database side.
# Replace schema name with a configured one, just in case it may be
# used by someone.
if schema_name:
schema.name = schema_name
metadata = MetaDataBuilder(schema).build()
else:
builder = MetaDataBuilder(schema, apply_schema_to_metadata=False, apply_schema_to_tables=False)
metadata = builder.build()

# Add table for replication support.
sqlalchemy.schema.Table(
"PpdbReplicaChunk",
metadata,
sqlalchemy.schema.Column(
"apdb_replica_chunk", sqlalchemy.BigInteger, primary_key=True, autoincrement=False
# Add replica chunk table.
table_name = "PpdbReplicaChunk"
columns = [
schema_model.Column(
name="apdb_replica_chunk",
id=f"#{table_name}.apdb_replica_chunk",
datatype=felis.datamodel.DataType.LONG,
),
schema_model.Column(
name="last_update_time",
id=f"#{table_name}.last_update_time",
datatype=felis.datamodel.DataType.TIMESTAMP,
nullable=False,
),
schema_model.Column(
name="unique_id",
id=f"#{table_name}.unique_id",
datatype=schema_model.ExtraDataTypes.UUID,
nullable=False,
),
schema_model.Column(
name="replica_time",
id=f"#{table_name}.replica_time",
datatype=felis.datamodel.DataType.TIMESTAMP,
nullable=False,
),
]
indices = [
schema_model.Index(
name="PpdbInsertId_idx_last_update_time",
id="#PpdbInsertId_idx_last_update_time",
columns=[columns[1]],
),
schema_model.Index(
name="PpdbInsertId_idx_replica_time",
id="#PpdbInsertId_idx_replica_time",
columns=[columns[3]],
),
sqlalchemy.schema.Column("last_update_time", sqlalchemy.types.TIMESTAMP, nullable=False),
sqlalchemy.schema.Column("unique_id", GUID, nullable=False),
sqlalchemy.schema.Column("replica_time", sqlalchemy.types.TIMESTAMP, nullable=False),
sqlalchemy.schema.Index("PpdbInsertId_idx_last_update_time", "last_update_time"),
sqlalchemy.schema.Index("PpdbInsertId_idx_replica_time", "replica_time"),
schema=schema_name,
]

# Add table for replication support.
chunks_table = schema_model.Table(
name=table_name,
id=f"#{table_name}",
columns=columns,
primary_key=[columns[0]],
indexes=indices,
constraints=[],
)
schema.tables.append(chunks_table)

if isinstance(schema.version, str):
version = VersionTuple.fromString(schema.version)
elif isinstance(schema.version, SchemaVersion):
if schema.version is not None:
version = VersionTuple.fromString(schema.version.current)
else:
# Missing schema version is identical to 0.1.0
version = VersionTuple(0, 1, 0)

metadata = sqlalchemy.schema.MetaData(schema=schema_name)

converter = ModelToSql(metadata=metadata)
converter.make_tables(schema.tables)

return metadata, version

@classmethod
Expand Down Expand Up @@ -322,7 +363,7 @@ def _make_engine(cls, config: PpdbSqlConfig) -> sqlalchemy.engine.Engine:
kw["poolclass"] = NullPool
if config.isolation_level is not None:
kw.update(isolation_level=config.isolation_level)
elif config.db_url.startswith("sqlite"): # type: ignore
elif config.db_url.startswith("sqlite"):
# Use READ_UNCOMMITTED as default value for sqlite.
kw.update(isolation_level="READ_UNCOMMITTED")
if config.connection_timeout is not None:
Expand Down
22 changes: 22 additions & 0 deletions python/lsst/dax/ppdb/tests/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# This file is part of dax_ppdb.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (http://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from ._ppdb import *
65 changes: 65 additions & 0 deletions python/lsst/dax/ppdb/tests/_ppdb.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# This file is part of dax_ppdb.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (http://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from __future__ import annotations

__all__ = ["PpdbTest"]

import unittest
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any

from ..config import PpdbConfig
from ..ppdb import Ppdb

if TYPE_CHECKING:

class TestCaseMixin(unittest.TestCase):
"""Base class for mixin test classes that use TestCase methods."""

else:

class TestCaseMixin:
"""Do-nothing definition of mixin base class for regular execution."""


class PpdbTest(TestCaseMixin, ABC):
"""Base class for Ppdb tests that can be specialized for concrete
implementation.
This can only be used as a mixin class for a unittest.TestCase and it
calls various assert methods.
"""

@abstractmethod
def make_instance(self, **kwargs: Any) -> PpdbConfig:
"""Make database instance and return configuration for it."""
raise NotImplementedError()

def test_empty_db(self) -> None:
"""Test for instantiation a database and making queries on empty
database.
"""
config = self.make_instance()
ppdb = Ppdb.from_config(config)
chunks = ppdb.get_replica_chunks()
if chunks is not None:
self.assertEqual(len(chunks), 0)
Loading

0 comments on commit 03d4c2d

Please sign in to comment.