feature/convert aXY examples part 1

noxfile.py

@@ -15,7 +15,13 @@
 
 def _test_command(path: Path) -> Iterable[str]:
  base_command = ["poetry", "run"]
- pytest_command = ["pytest", "-v", f"{path}"]
+ pytest_command = [
+ "pytest",
+ "-v",
+ "--log-level=INFO",
+ "--log-cli-level=INFO",
+ f"{path}"
+ ]
  return base_command + pytest_command
 
 
@@ -80,5 +86,5 @@ def integration_tests(session: Session) -> None:
 @ nox.session(name="all-tests", python=False)
 def all_tests(session: Session) -> None:
  """Runs all tests (Unit and Integration)"""
- command=_test_command(_ROOT / "test")
+ command = _test_command(_ROOT / "test")
  session.run(*command)

pyproject.toml

@@ -63,6 +63,7 @@ nox = "^2023.4.22"
 pytest = ">=7.0.0,<9"
 docutils = "0.20.1"
 exasol-integration-test-docker-environment = "^3.0.0"
+faker = "^24.14.1"
 
 [build-system]
 requires = ["poetry-core"]

scripts/generate_users.py

@@ -0,0 +1,98 @@
+import sys
+import csv
+import decimal
+import random
+import datetime
+import argparse
+
+from faker import Faker
+from faker.providers import BaseProvider
+
+
+class UserDataProvider(BaseProvider):
+
+ def date(self):
+ start_date = datetime.date(2018, 1, 1)
+ return start_date + datetime.timedelta(random.randint(1, 365))
+
+ def timestamp(self):
+ date = self.date()
+ time = datetime.time(
+ random.randint(0, 23),
+ random.randint(0, 59),
+ random.randint(0, 59),
+ random.randint(0, 999) * 1000
+ )
+ return datetime.datetime.combine(date, time)
+
+ def boolean(self):
+ return self.random_element([True, False])
+
+ def status(self):
+ return self.random_element(
+ ['ACTIVE', 'PENDING', 'SUSPENDED', 'DISABLED']
+ )
+
+ def decimal(self):
+ return decimal.Decimal(random.randint(0, 100)) / 100
+
+ def score(self):
+ value = random.randint(0, 10)
+ return None if value == 10 else random.randint(0, 10000) / 100
+
+
+def generate_users(count):
+ fake = Faker()
+ fake.add_provider(UserDataProvider)
+
+ for i in range(count):
+ yield (
+ i,
+ fake.name(),
+ fake.date(),
+ fake.timestamp(),
+ fake.boolean(),
+ fake.decimal(),
+ fake.score(),
+ fake.status()
+ )
+
+
+def _create_parser():
+ parser = argparse.ArgumentParser(
+ description="Generate a CSV file containing users",
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter
+ )
+ parser.add_argument(
+ 'filename',
+ type=argparse.FileType('w', encoding='utf8'),
+ help='file the resulting CSV should be written to.'
+ )
+ parser.add_argument(
+ '-n', '--count',
+ type=int, default=10000,
+ help='Number of users to create.'
+ )
+
+ return parser
+
+
+FAILURE = -1
+SUCCESS = 0
+
+
+def main(argv=None):
+ parser = _create_parser()
+ args = parser.parse_args(argv)
+ try:
+ writer = csv.writer(args.filename, delimiter=',')
+ for user in generate_users(count=args.count):
+ writer.writerow(user)
+ except Exception as ex:
+ print(f"Error while generating users, details: {ex}")
+ return FAILURE
+ return SUCCESS
+
+
+if __name__ == '__main__':
+ sys.exit(main())

test/data/import.sql

@@ -0,0 +1,18 @@
+DROP SCHEMA IF EXISTS PYEXASOL_TEST CASCADE;
+CREATE SCHEMA PYEXASOL_TEST;
+
+CREATE OR REPLACE TABLE PYEXASOL_TEST.USERS
+(
+ user_id DECIMAL(18,0),
+ user_name VARCHAR(255),
+ register_dt DATE,
+ last_visit_ts TIMESTAMP,
+ is_female BOOLEAN,
+ user_rating DECIMAL(10,5),
+ user_score DOUBLE,
+ status VARCHAR(50)
+);
+
+IMPORT INTO PYEXASOL_TEST.USERS FROM LOCAL CSV FILE 'users.csv' COLUMN SEPARATOR = ',';
+
+

test/integration/conftest.py

@@ -1,26 +1,31 @@
 import os
+import uuid
 import pytest
 import pyexasol
+import subprocess
+from pathlib import Path
 
+import logging
 
-@pytest.fixture
+
+@pytest.fixture(scope='session')
 def dsn():
  return os.environ.get('EXAHOST', 'localhost:8563')
 
 
-@pytest.fixture
+@pytest.fixture(scope='session')
 def user():
  return os.environ.get('EXAUID', 'SYS')
 
 
-@pytest.fixture
+@pytest.fixture(scope='session')
 def password():
  return os.environ.get('EXAPWD', 'exasol')
 
 
-@pytest.fixture
+@pytest.fixture(scope='session')
 def schema():
- return os.environ.get('EXASCHEMA', 'TEST')
+ return os.environ.get('EXASCHEMA', 'PYEXASOL_TEST')
 
 
 @pytest.fixture
@@ -29,6 +34,135 @@ def connection(dsn, user, password, schema):
  dsn=dsn,
  user=user,
  password=password,
+ schema=schema
  )
  yield con
  con.close()
+
+
+@pytest.fixture(scope='session', autouse=True)
+def prepare_database(dsn, user, password):
+ class DockerDataLoader:
+ """Data loader for docker based Exasol DB"""
+
+ def __init__(
+ self,
+ dsn,
+ username,
+ password,
+ container_name,
+ data_directory
+ ):
+ self._logger = logging.getLogger("DockerDataLoader")
+ self._dsn = dsn
+ self._user = username
+ self._password = password
+ self._container = container_name
+ self._data_directory = data_directory
+ self._tmp_dir = f"data-{uuid.uuid4()}"
+
+ @property
+ def data_directory(self):
+ return self._data_directory
+
+ @property
+ def ddl_file(self):
+ return self.data_directory / "import.sql"
+
+ @property
+ def csv_files(self):
+ return self.data_directory.rglob("*.csv")
+
+ def load(self):
+ self._create_dir()
+ self._upload_files()
+ self._import_data()
+
+ def _execute_command(self, command):
+ self._logger.info("Executing docker command: %s", command)
+ result = subprocess.run(
+ command,
+ check=True,
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+ self._logger.debug("Stderr: %s", result.stderr)
+ return result.stdout
+
+ def _exaplus(self) -> str:
+ find_exaplus = [
+ "docker",
+ "exec",
+ self._container,
+ "find",
+ "/usr",
+ "-name",
+ "exaplus",
+ "-type",
+ "f", # only files
+ "-executable", # only executable files
+ "-print", # -print -quit will stop after the result is found
+ "-quit",
+ ]
+ exaplus = self._execute_command(find_exaplus).strip()
+ self._logger.info("Found exaplus at %s", exaplus)
+ return exaplus
+
+ def _create_dir(self):
+ """Create data directory within the docker container."""
+ mkdir = [
+ "docker",
+ "exec",
+ self._container,
+ "mkdir",
+ self._tmp_dir
+ ]
+ stdout = self._execute_command(mkdir)
+ self._logger.info("Stdout: %s", stdout)
+
+ def _upload_files(self):
+ files = [self.ddl_file]
+ files.extend(self.csv_files)
+ for file in files:
+ copy_file = [
+ "docker",
+ "cp",
+ f"{file.resolve()}",
+ f"{self._container}:{self._tmp_dir}/{file.name}",
+ ]
+ stdout = self._execute_command(copy_file)
+ self._logger.debug("Stdout: %s", stdout)
+
+ def _import_data(self):
+ """Load test data into a backend."""
+ execute_ddl_file = [
+ "docker",
+ "exec",
+ "-w",
+ f"/{self._tmp_dir}",
+ self._container,
+ self._exaplus(),
+ "-c",
+ f"{self._dsn}",
+ "-u",
+ self._user,
+ "-p",
+ self._password,
+ "-f",
+ self.ddl_file.name,
+ "--jdbcparam",
+ "validateservercertificate=0",
+ ]
+ stdout = self._execute_command(execute_ddl_file)
+ self._logger.info("Stdout: %s", stdout)
+
+ data_directory = Path(__file__).parent / '..' / 'data'
+ loader = DockerDataLoader(
+ dsn=dsn,
+ username=user,
+ password=password,
+ container_name='db_container_test',
+ data_directory=data_directory
+ )
+ loader.load()

test/integration/integration_smoke_test.py

@@ -1,5 +1,19 @@
-def test_smoke(connection):
+def test_static_select(connection):
  result = connection.execute("SELECT 1;")
- expected = (1,)
- actual = result.fetchall()[0]
+ expected = [(1,)]
+ actual = result.fetchall()
+ assert expected == actual
+
+
+def test_sorted_select_and_limited_select(connection):
+ statement = f"SELECT * FROM USERS ORDER BY USER_ID LIMIT 5;"
+ result = connection.execute(statement)
+ expected = [
+ (0, 'Amy Marquez', '2018-10-04', '2018-03-06 21:44:36.142000', True, '0.76', 30.11, 'PENDING\r'),
+ (1, 'John Lawson', '2018-05-17', '2018-05-28 02:58:29.079000', True, '0.04', 71.72, 'DISABLED\r'),
+ (2, 'Jessica Clark', '2018-05-23', '2018-05-22 04:19:51.098000', False, '0.72', 29.13, 'PENDING\r'),
+ (3, 'Jennifer Taylor', '2018-05-01', '2018-03-03 08:12:52.685000', True, '0.43', 8.46, 'SUSPENDED\r'),
+ (4, 'Tristan Romero', '2018-10-04', '2018-03-31 20:21:50.199000', True, '0.23', 62.980000000000004, 'PENDING\r')
+ ]
+ actual = result.fetchall()
  assert expected == actual