diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..1f4911f3 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,25 @@ +name: Lint + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Lint with ruff + run: | + pip install ruff + ruff check python/ diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml deleted file mode 100644 index 31f6447d..00000000 --- a/.github/workflows/pythonapp.yml +++ /dev/null @@ -1,40 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python application - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - build: - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install . - - - name: Lint with flake8 - run: | - pip install flake8 - flake8 python/ - - - name: Test basic import - run: | - python -c "from sdssdb.connection import DatabaseConnection" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..56562f10 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,51 @@ +name: Test + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + name: Test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Set up uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Restore uv cache + uses: actions/cache@v4 + with: + path: /tmp/.uv-cache + key: uv-${{ runner.os }}-${{ hashFiles('uv.lock') }} + restore-keys: | + uv-${{ runner.os }}-${{ hashFiles('uv.lock') }} + uv-${{ runner.os }} + + - name: Install Postgresql + uses: ikalnytskyi/action-setup-postgres@v6 + with: + username: postgres + id: postgres + + - name: Install dependencies + run: | + uv pip install --system -e .[dev] + + - name: Test with pytest + run: | + pytest -s + env: + PGSERVICE: ${{ steps.postgres.outputs.service-name }} + + - name: Minimize uv cache + run: uv cache prune --ci diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 1a2c0f7c..00000000 --- a/.pylintrc +++ /dev/null @@ -1,425 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=C,R,W0703,W0221,W0511,W0212,W0621,import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable= - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=99 - -# Maximum number of lines in a module -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[BASIC] - -# Naming hint for argument names -argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct argument names -argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for attribute names -attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct attribute names -attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming hint for function names -function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct function names -function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for method names -method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct method names -method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming hint for variable names -variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 00cae40b..00000000 --- a/.travis.yml +++ /dev/null @@ -1,43 +0,0 @@ -language: python - -cache: - pip: true - timeout: 1000 - -services: - - postgresql - -python: -- '3.6' -- '3.7' -- '3.8' - -os: -- linux - -matrix: - fast_finish: true - allow_failures: - - python: '3.6' - - -notifications: - email: false - -# repo branches to test -branches: -- main - -install: -- pip install -U pip wheel --quiet -- pip install --upgrade setuptools --quiet -- pip install pytest -- pip install pytest-coverage -- pip install coveralls -- pip install .[dev] - -script: -- pytest -p no:sugar tests --cov python/sdssdb --cov-report html - -after_success: -- coveralls diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cec752e2..e4c8ddf8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,7 @@ Changelog This document records the main changes to the ``sdssdb`` code. * :feature:`264` Add metadata for the ``sdss_id_to_catalog`` table. +* :feature:`266` Support PEP 621. * Add missing columns to the ``SDSS_ID_To_Catalog`` models (the columns were being completed via reflection). * Change ``reflection=False`` to ``use_reflection=False`` in many ``catalogdb`` models. The previous setting was incorrect and was causing the models to be reflected. * Add ``gortdb`` schema and models to ``lvmdb``. Moved ``lvmopsdb.overhead`` to ``gortdb``. diff --git a/CODEOWNERS b/CODEOWNERS index 2c08e4ea..ba4b949e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -7,6 +7,7 @@ # review when someone opens a pull request. * @albireox * gallegoj@uw.edu +* bcherinka@stsci.edu # Order is important; the last matching pattern takes the most # precedence. When someone opens a pull request that only diff --git a/STYLE.rst b/STYLE.rst deleted file mode 100644 index d5f2839f..00000000 --- a/STYLE.rst +++ /dev/null @@ -1,527 +0,0 @@ -SDSS Python template and coding standards -========================================= - -So you want to write some Python code. Congratulations, you've arrived at -the right place! This repository has a dual purpose: it provides a -`template `__ for a basic but complete Python package; and lists the coding -standards and recommendations for developing code for SDSS. Please, read -this document carefully. If you decide to develop your product based on -this template, feel free to replace ``__ with a description -of your project, but keep the ``__ file as a reminder of the -coding conventions. - -While this document deals with Python product, and some of the solutions -and services suggested are specific for it, much of what is written here -is general good advice for developing software in any platform. - -Table of Contents ------------------ - -- `Python 2 vs Python 3: which one to - choose? <#python-2-vs-python-3-which-one-to-choose>`__ -- `Code storage and ownership. <#code-storage-and-ownership>`__ -- `Tagging, versioning, and change logs. <#tagging-versioning-and-change-logs>`__ -- `Deployment <#deployment>`__ -- `Coding style <#coding-style>`__ - - - `Docstrings <#docstrings>`__ - - `Linters <#linters>`__ - - `General advice <#general-advice>`__ - -- `Testing <#testing>`__ - - - `Unit testing <#unit-testing>`__ - - `Continuous integration and - coverage <#continuous-integration-and-coverage>`__ - -- `Automatic documentation - generation <#automatic-documentation-generation>`__ - - - `Read the Docs <#read-the-docs>`__ - -- `Git workflow <#git-workflow>`__ -- `Software Citation <#software-citation>`__ - - - `Zenodo <#zenodo>`__ - - `Astrophysical Source Code Library <#ascl>`__ - - -Python 2 vs Python 3: which one to choose? ------------------------------------------- - -SDSS has made the decision to transition to Python 3 by 2020. That means -that all new code must *at least* be compatible with Python 3.6. There is, -however, a very significant amount of ancillary code that is still -Python 2-only and that will not be ported to Python 3 for some time. - -When deciding what version of Python to write your code on, consider -which are its dependencies: - -- If your code is standalone, or depends on Python 3-compatible code, - write it in Python 3. **You don't need to make sure your code is - Python 2-backwards compatible.** - -- If your code depends on key packages that are Python 2-only (e.g., - ``actorcore``, ``opscore``, ``RO``, ``twistedActor``), write your - code in Python 2 **but** try to make it as much Python 3-ready as - possible, so that when those dependencies are upgraded you can - upgrade your code easily. - -- If your code is intended for a large user base, Python 2 and 3 compatibility is recommended, but the focus should be put into Python 3. - -Whenever you create a new Python file, make sure to add the following -lines at the top of the file - -.. code:: python - - from __future__ import division - from __future__ import print_function - from __future__ import absolute_import - from __future__ import unicode_literals - -That will force you to use ``import``, ``print``, and division in a way -that is Python 2 and 3-compatible. - -Some resources that can be useful to write code that is Python 2 and -3-compatible, and to port code from 2 to 3: - -- A `cheat sheet `__ - with advice to write code compatible with Python 2 and 3. -- The `six `__ library provides - functions to write code that will work in Python 2 and 3. -- When converting code from Python 2 to 3, consider using - `python-futurize `__ as the - starting point. It works very well for most files, and even for those - files that require manual interaction, it paves most of the way. - -Code storage and ownership --------------------------- - -All code must be version controlled using -`git `__. Older code, still under the SVN -repository, can be maintained using Subversion until it has been ported -to Git. Large data-only repositories that use SVN do not need to be ported. - -All code must live in the `SDSS GitHub -organisation `__. Code that is specific to Apache Point Observatory and it is shared with other on-site telescopes should be put in `their own `__ organisation. When starting a new -product, start a new repository in the GitHub organisation (you can -choose to make it public or private) and follow the instructions to -clone it to your computer. Feel free to create forks of the repositories -to your own GitHub account, but make sure the production version of the -code lives in the organisation repo. - -If your code is already in GitHub, move it to the SDSS GitHub organisation as soon as it is ready to be shared. This can be done easily by creating a new repository in the SDSS GitHub, adding it as a new remote to your local checkout, and pushing to the new remote. - -All code must have *at least* one owner, who is ultimately responsible -for keeping the code working and making editorial decisions. Owners can -make decisions on which code standards to follow (within the requirements -listed in this document), such as maximum line length, linter, or -testing framework. The owner(s) names should be obvious in the README of -the repo and in the ``setup.py`` file. - -Tagging, versioning, and change logs ------------------------------------ - -**All production software must run from tagged versions.** The only exception to this rule is when debugging new code during engineering or test runs. - -Following `PEP 440 `__, software versions should use the convention ``X.Y.Z`` (e.g., -``1.2.5``) where X indicates the major version (large, maybe -non-backwards compatible changes), Y is for minor changes and additions -(backwards compatible), and Z is for bug fixes (no added functionality). -Suffixes to the version, such as ``dev``, ``alpha``, ``beta``, are -accepted. Do not use a hyphen between version and suffix (``1.2.5dev`` -is ok, ``1.2.5-dev`` is not). Note that `PEP 440 `__ recommends separating suffixes with a period (``1.2.5.dev``) but we have found that sometimes causes problems with pip. - -For products that already have tagged versions using the old SDSS versioning standards (e.g., ``v1_2_3``), tag new versions using the new convention (e.g., ``1.2.4``) but do not rename or retag previous versions. - -Python packages must return its version via the ``__version__`` attribute. All other products, including metadata and datamodels, must also be versioned in a clear and obvious way. When tagging using git, prefer `annotated tags `__. - -Version tracking may be complicated so we recommend using -``bumpversion`` (see `here `__ -for documentation). This template already implements a `configuration -file <./.bumpversion.cfg>`__ that automates updating the version number -in all the places in the code where it appears. Let's say that your -current version is ``0.5.1`` and you are going to work on minor changes -to the product. You can go to the root of the package and run -``bumpversion minor``. This will update the version to ``0.6.0dev`` -everywhere needed, and will commit the changes. When you are ready to -release, you can do ``bumpversion release`` to change the version to -``0.6.0``. See the `template documentation `__ for more details. - -All files must include in their metadata the version of the software that produced them, along with the versions of all relevant dependencies. For instance, data FITS must include the version of the pipeline in the header. - -All changes should be logged in a ``CHANGELOG.rst`` or ``CHANGELOG.md`` -file. See `the template CHANGELOG.rst <./CHANGELOG.rst>`__ for an -example of formatting. When releasing a new version, copy the change log -for the relevant version in the GitHub release description. - -Deployment ----------- - -SDSS Python packages should follow the general Python standards for -packaging. If looking for documentation, `start -here `__. - -All packages must contains a `setup.py <./setup.py>`__ to automate -building, installation, and packaging. The ``setup.py`` file must take -care of compiling and linking all external code (e.g., C libraries) that -is used by the project. - -Dependencies must be maintained in two different locations. For -standard, pip-installable dependencies, use the -`requirements.txt <./requirements.txt>`__ file. See -`here `__ -for more information on using requirements.txt files. Consider using -multiple requirements.txt files (e.g, ``requirements.txt``, -``requirements_dev.txt``, ``requirements_docs.txt``) for different -pieces of functionality. Additionally, you must maintain the -`module `__ file for your product. If you -package depends on SDSS-specific, non pip-installable packages, use the -module file to load the necessary dependencies. - -Should you make your package pip-installable? The general answers is -yes, but consider the scope of your project. If your code is to be used -for mountain operations and needs to be maintained with modules/EUPS -version control, making it pip installable may not be necessary, since -it is unlikely to be installed in that way. However, if your product -will be distributed and installed widely in the collaboration (examples -of this include analysis tools, pipelines, schedulers), you *must* make -it pip-installable. Start `here `__ for -some documentation on making pip-installable packages. Another good -resource is `twine `__, which will help -you automate much of the packaging and uploading process. - -SDSS has a `PyPI account `__ that should be -used to host released version of your pip-installable projects. Do not -deploy the project in your own account. Instead, contact -``admin[at]sdss[dot]org`` to get access to the PyPI account. - -Coding style ------------- - -SDSS code follows the `PEP8 -standard `__. Please, read -that document carefully and follow every convention, unless there are -very good reasons not to. - -The only point in which SDSS slightly diverges from PEP8 is the line -length. While the suggested PEP8 maximum line length of 79 characters is -recommended, lines **up to** 99 characters are accepted. When deciding -what line length to use, follow this rule: if you are modifying code -that is not nominally owned by you, respect the line length employed by -the owner of the product; if you are creating a new product that you -will own, feel free to decide your line length, as long as it has fewer -than 99 characters. - -It is beyond the scope of this document to summarise the PEP8 -conventions, but here are some of the most salient points: - -- Indentation of four spaces. **No tabs. Ever.** -- Two blank lines between functions and classes. One blank line between - methods in a class. A single line at the end of each file. -- Always use spaces around operators and assignments (``a = 1``). The - only exception is for function and method keyword arguments - (``my_function(1, key='a')``). -- No trailing spaces. You can configure your editor to strip the lines - automatically for you. -- Imports go on the top of the file. Do **not** import more than one - package in the same line (``import os, sys``). Maintain the - namespace, do **not** import all functions in a package - (``from os import *``). You can import multiple functions from the - same package at the same time - (``from os.path import dirname, basename``). -- Use single quotes for strings. Double quotes must be reserved for - docstrings and string blocks. -- For inline comments, at least two spaces between the statement and - the beginning of the comment - (``a = 1­­  # This is a comment about a``). -- Class names must be in camelcase (``class MyClass``). Function, - method, and variable names should be all lowercase separated by - underscores for legibility (``def a_function_that_does_something``, - ``my_variable = 1``). For the latter ones, PEP8 allows some - flexibility. The general rule of thumb is to make your function, - method, and variable names descriptive and readable (avoid multiple - words in all lowercase). As such, if you prefer to use camelcase - (``aFunctionThatDoesSomething``, ``myVariable = 1``) for your project - that is accepted, as long as you are consistent throughout the - project. When modifying somebody else's code, stick to their naming - decisions. -- Use ``is`` for comparisons with ``None``, ``True``, or ``False``: - ``if foo is not None:``. - -.. _style-docstring: - -Docstrings -~~~~~~~~~~ - -Docstrings are special comments, wrapped between two sets of three -double quotes (``"""``). Their purpose is dual: on one side they provide -clear, well structured documentation for each class and function in your -code. But they are also intended to be read by an automatic -documentation generator (see the `Automatic documentation -generation <#automatic-documentation-generation>`__ section). For -docstrings, follow -`PEP257 `__. In our template, -`main.py <./python/python_template/main.py>`__ contains some examples of -functions and classes with docstrings; use those as an example. In -general: - -- **All** code should be commented. **All** functions, classes, and - methods should have a docstring. -- Use double quotes for docstrings; reserve single quotes for normal - strings. -- Limit your docstrings lines to 72 characters. This convention can be a bit constraining for some developers; it is ok to ignore it and use the line length you are using for your code (79 or 99 characters). -- A complete docstring should start with a single line describing the - general purpose of the function or class. Then a blank line and an - in-depth description of the function or class in one or more - paragraphs. A list of the input parameters (arguments and keywords) - follows, and a description of the values returned, if any. If the - class or function merits it, you should include an example of use. -- The docstring for the ``__init__()`` method in a class goes just - after the declaration of the class and it explains the general use - for the class, in addition to the list of parameters accepted by - ``__init__()``. -- Private methods and functions (those that start with an underscore) - may not have a docstring **only** if their purpose is really obvious. -- In general, we prefer `Google - style `__ - docstrings over `Numpy - style `__ - ones, but you are free to choose one as long as you stick with it - across all the product. Avoid styles such as - ``param path: The path of the file to wrap`` which are difficult to - read. - -Linters -~~~~~~~ - -Do use a linter. These are plugins available for almost every editor -(vim, emacs, Sublime Text, Atom) that are executed every time you save -your code and show you syntax errors and where you are not following -PEP8 conventions. They normally rely on an underlying library, usually -`pylint `__ or -`flake8 `__. This template includes -customised configuration files for both libraries. You can also place -``.flake8`` and ``.pylintrc`` files in your home directory and they will -be used for all your projects (configuration files *in* the root of the -project override the general configuration for that project). - -While ``pylint`` is a more fully fleshed library, and provides estimates -on code complexity, docstring linting, etc., it may be a bit excessive -and verbose for most users. ``flake8`` provides more limited features, -but its default configuration is usually what you want (and we enforce -in SDSS). It is up to you to test them and decide which one to use. - -Do update the ``.flake8`` or ``.pylintrc`` files in your project with -the specific configuration you want to use in for that product. That is -critical for other people to contribute to the code while keeping your -coding style choices. - -File headers -~~~~~~~~~~~~ - -Include a header in each Python file describing the author, license, -etc. We suggest - -.. code:: python - - # encoding: utf-8 - # - # @Author: - # @Date: - # @Filename: - # @License: - # @Copyright: - - - from __future__ import division - from __future__ import print_function - from __future__ import absolute_import - from __future__ import unicode_literals - -In general, do not include comments about when you last modified the -file since those become out of date really fast. Instead, use the `changelog <./CHANGELOG.rst>`__ and atomic git -commits. - -All executable files should live in the ``bin/`` directory. For those files, add a shebang at the beginning of the header :: - - #!/usr/bin/env python - -General advice -~~~~~~~~~~~~~~ - -- Blank lines only add one byte to your file size; use them prolifically to improve legibility. -- Read the `Zen of Python `__. Explicit is better than implicit. Simple is better than complex. -- Know when ignore these standards if there is a good reason or it improves readability (but don't use that as an excuse to just not follow the standards). - -Testing -------- - -Do test your code. Do test your code. Do test your code. As repository -owner, you are the ultimate responsible for making sure your code does -what it is supposed to do, and to avoid that new features break current -functionality. - -Modern testing standards are based on two cornerstone ideas: `unit -testing `__, and `continuous -integration `__ -(CI). - -Unit testing -~~~~~~~~~~~~ - -Unit testing advocates for breaking your code into small "units" that -you can write tests for (and then actually write the tests!) There are -multiple tutorials and manuals online, `this -one `__ is a good -starting point. - -Many libraries and frameworks for testing exist for Python. The basic -(but powerful) one is called -`unittest `__ and is a -standard Python library. -`nose2 `__ provides additional -features, and a nicer interface. -`pytest `__ includes all those extra -features plus a number of extremely convenient and powerful features, as -well as many third-party addons. On the other hand, its learning curve -may be a bit steep. - -So, what library should you use? If your code and testing needs are very -simple, ``unittest`` is a good option. - -For larger projects, SDSS recommends using ``pytest``. Features such as -`parametrising -tests `__ -and `fixtures `__ are -excellent to make sure your code gets a wide test coverage. This -template includes a simple `pytest -setup <./python/python_template/test>`__. You can also look at the -`Marvin test -suite `__ -for a more complete example. - -Continuous integration and coverage -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is critical that you not only write test but run them, and do so in a -suite of environments (different OS, Python versions, etc). Doing that -in your local computer can be convoluted, so we recommend the use of -`Travis CI `__. Travis gets integrated with a -GitHub repository and is triggered every time you commit, make a pull -request, or create a branch. On trigger, you can configure what happens -before the tests are run (e.g, download files, create a database), and -the platforms they run on. For an example of a full Travis setup see the -`Marvin travis -configuration `__. - -In addition to running tests, you will want to keep an eye on test -coverage, i.e., what percentage of your code gets "activated" and tested -with your unit tests. Increasing your test coverage should always be a -goal, as it is to make sure that any new feature or bug fix gets -associated tests. You can check your coverage using -`pytest-cov `__. -`Coveralls `__ is another CI service that can be -configured to run after Travis and that provides a nice HTML display of -your coverage and missing lines. - -Automatic documentation generation ----------------------------------- - -As a software developer, it is part of your responsibility to document -your code and keep that documentation up to date. Documentation takes -two forms: inline documentation in the form of comments and docstrings; -and explicit documentation, tutorials, plain-text explanations, etc. - -Explicit documentation can take many forms (PDFs, wiki pages, plain text -files) but the rule of thumb is that the best place to keep your -documentation is the product itself. That makes sure a user knows where -to look for the documentation, and keeps it under version control. - -SDSS uses and **strongly encourages** -`Sphinx `__ to -automatically generate documentation. Sphinx translates -`reStructuredText `__ source -files to HTML (plugins for Latex, HTML, and other are available). It -also automates the process of gathering the docstrings in your code and -generating nicely formatted HTML code. - -It is beyond the purpose of this document to explain how to use Sphinx, -but `its -documentation `__ is -quite good and multiple tutorials exist online. A large ecosystem of -plugins and extensions exist to perform almost any imaginable task. This -template includes a basic but functional `Sphinx -template <./docs/sphinx>`__ that you can build by running ``make html``. - -Read the Docs -~~~~~~~~~~~~~ - -Deploying your Sphinx documentation is critical. SDSS uses `Read the -Docs `__ to automatically build and deploy -documentation. Read the Docs can be added as a plugin to your GitHub -repo for continuous integration so that documentation is built on each -commit. SDSS owns a Read the Docs account. Contact -``admin[at]sdss[dot]org`` to deploy your documentation there. Alternatively, you can deploy your product in your own Read the Docs account and add the user ``sdss`` as a maintainer from the admin menu. - -Git workflow ------------- - -Working with Git and GitHub provides a series of extremely useful tools -to write code collaboratively. Atlassian provides a `good -tutorial `__ on Git -workflows. While the topic is an extensive one, here is a simplified -version of a typical Git workflow you should follow: - -1. `Clone `__ the repository. -2. Create a `branch `__ (usually - from master) to work on a bug fix or new feature. Develop all your - work in that branch. Commit frequently and modularly. Add tests. -3. Once your branch is ready and well tested, and your are ready to - integrate your changes, you have two options: - - 1. If you are the owner of the repo and no other people are - contributing code at the time (or your changes are **very** small - and non-controversial) you can simple - `merge `__ the branch back - into master and push it to the upstream repo. - 2. If several people are collaborating in a project, you *want* to - create a `pull - request `__ - for that branch. The change can then be discussed, changes made - and, when approved, you can merge the pull request. - -4. GOTO 2 - -You may want to consider the possibility of using -`forks `__ if you are -planning on doing a large-scope change to the code. - -Software Citation ------------------ - -All software should be archived and citable in some way by anyone who uses it. The AAS now has a -policy for `software citation `_, that SDSS should adopt -for all pieces of code it produces. This policy should be adopted by internal SDSS collaborators -as well as astronomers outside SDSS using SDSS software. - -Zenodo -~~~~~~ - -Zenodo allows you to generate a unique digital object identifier (DOI) for any piece of software code in a GitHub -repository. DOI's are citable snippets, and allow your software code to be identified by tools. See `Making Your Code Citable `_ for how to connect your GitHub repository to Zenodo. Once your GitHub repo is connected to Zenodo, every new GitHub tag or release gets a new DOI from Zenodo. Zenodo provides a citable formats for multiple journals as well as export to a Bibtex file. - -Astrophysical Source Code Library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The `ASCL `_ is a registry of open-source astronomy software, indexed by the -`SAO/NASA Astrophysics Data System `_ (ADS). The process for submission -to the ASCL is outlined `here `_. - -Further reading ---------------- - -- Python's own `documentation style - guide `__ is a - good resource to learn to write good documentation. -- Astropy's `coding standards `__ and `documentation guide `__ are good resources. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..e94d5dd3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,149 @@ +[project] +name = "sdssdb" +version = "0.12.5a0" +authors = [ + {name = "José Sánchez-Gallego", email = "gallegoj@uw.edu"}, + {name = "Brian Cherinka", email = "bcherinka@stsci.edu"} +] +description = "SDSS product for database management" +keywords = ["astronomy", "software", "database"] +license = {text = "BSD 3-Clause License"} +requires-python = ">=3.6" +dependencies = [ + "pgpasslib>=1.1.0", + "psycopg2-binary>=2.7.7", + "peewee>=3.17.6", + "sqlalchemy>=1.3.6,<2", + 'sdsstools>=1.7.0; python_version >= "3.8"', + 'sdsstools>=0.1.0,<1.0.0; python_version < "3.8"', + "numpy>=1.18.2", +] + +[project.urls] +Homepage = "https://github.com/sdss/sdssdb" +Repository = "https://github.com/sdss/sdssdb" +Documentation = "https://sdssdb.readthedocs.org" + +[project.readme] +file = "README.md" +content-type = "text/markdown" + +[project.optional-dependencies] +all = [ + "progressbar2>=3.46.1", + "pydot>=1.4.1", + "astropy>=4.0.0", + "pandas>=1.0.0", + "inflect>=4.1.0", + "h5py>=3.8.0", + "sdssdb[dev]", + "sdssdb[docs]", +] +dev = [ + "pytest>=5.2", + "pytest-cov>=2.4.0", + "pytest-sugar>=0.8.0", + "ipython>=7.13.0", + "ipdb>=0.13.2", + "pytest-postgresql>=2.2.1,<6", + "factory_boy>=2.12.0", + "pytest-factoryboy>=2.0.3", + "astropy>=4.0.0", + "pydot>=1.4.2", + "pyyaml>=5.1", + "ruff>=0.6.3", + "flake8>=7.1.1", + "flake8-pyproject>=1.2.3" +] +docs = [ + "Sphinx>=7.0.0", + "sphinx_bootstrap_theme>=0.4.12", + "releases>=2.0.0", +] + +[build-system] +requires = ["setuptools>=61.2"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +zip-safe = false +package-dir = {"" = "python"} +license-files = ["LICENSE.md"] +include-package-data = false + +[tool.setuptools.packages.find] +where = ["python"] +exclude = ["*.tests"] +namespaces = false + +[tool.setuptools.package-data] +sdssdb = ["etc/*"] + +[tool.isort] +line_length = 79 +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "SQLA", + "PEEWEE", + "SDSS", + "FIRSTPARTY", + "LOCALFOLDER", +] +default_section = "THIRDPARTY" +known_first_party = ["sdssdb"] +known_sqla = ["sqlalchemy"] +known_peewee = [ + "peewee", + "playhouse", +] +known_sdss = ["sdsstools"] +balanced_wrapping = true +include_trailing_comma = false +lines_after_imports = 2 +use_parentheses = true + +[tool.flake8] +ignore = [ + "H101", + "E722", + "W503", + "W504", + "W505", + "E116", + "E114" +] +max-line-length = 99 +exclude = [ + "python/sdssdb/sqlalchemy/operationsdb/tools", + "python/sdssdb/sqlalchemy/archive", + "python/sdssdb/sqlalchemy/mangadb" +] + +[tool.ruff] +line-length = 99 +target-version = 'py312' +exclude = ["typings/"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] +exclude = [ + "python/sdssdb/sqlalchemy/operationsdb/**/*.py", + "python/sdssdb/sqlalchemy/archive/*.py", + "python/sdssdb/sqlalchemy/mangadb/*.py" +] + +[tool.ruff.lint.isort] +known-first-party = ["sdssdb"] +lines-after-imports = 2 +section-order = ["future", "standard-library", "typing", "third-party", "sqla", "peewee", "sdss", "first-party", "local-folder"] + +[tool.ruff.lint.isort.sections] +typing = ["typing"] +sdss = ["sdsstools"] +sqla = ["sqlalchemy"] +peewee = ["peewee", "playhouse"] + +[tool.coverage.run] +branch = true diff --git a/python/sdssdb/__init__.py b/python/sdssdb/__init__.py index 94eb9702..8c841df5 100644 --- a/python/sdssdb/__init__.py +++ b/python/sdssdb/__init__.py @@ -2,22 +2,26 @@ import warnings +from sqlalchemy.exc import MovedIn20Warning + from sdsstools import get_config, get_logger, get_package_version +warnings.filterwarnings("ignore", category=MovedIn20Warning) + warnings.filterwarnings( - 'ignore', - '.*Skipped unsupported reflection of expression-based index .*q3c.*', + "ignore", + ".*Skipped unsupported reflection of expression-based index .*q3c.*", ) warnings.filterwarnings( - 'ignore', - '.*invalid escape sequence.*', + "ignore", + ".*invalid escape sequence.*", category=SyntaxWarning, ) -NAME = 'sdssdb' +NAME = "sdssdb" __version__ = get_package_version(path=__file__, package_name=NAME) diff --git a/python/sdssdb/connection.py b/python/sdssdb/connection.py index 9e46de7f..fa971a4c 100644 --- a/python/sdssdb/connection.py +++ b/python/sdssdb/connection.py @@ -14,30 +14,33 @@ import re import socket -import peewee import pgpasslib import six -from peewee import OperationalError, PostgresqlDatabase -from playhouse.postgres_ext import ArrayField -from playhouse.reflection import Introspector, UnknownField + from sqlalchemy import MetaData, create_engine from sqlalchemy.engine import url from sqlalchemy.exc import OperationalError as OpError from sqlalchemy.orm import scoped_session, sessionmaker +import peewee +from peewee import OperationalError, PostgresqlDatabase +from playhouse.postgres_ext import ArrayField +from playhouse.reflection import Introspector, UnknownField + import sdssdb from sdssdb import config, log from sdssdb.utils.internals import get_database_columns -__all__ = ['DatabaseConnection', 'PeeweeDatabaseConnection', 'SQLADatabaseConnection'] + +__all__ = ["DatabaseConnection", "PeeweeDatabaseConnection", "SQLADatabaseConnection"] def _should_autoconnect(): """Determines whether we should autoconnect.""" - if 'SDSSDB_AUTOCONNECT' in os.environ: - envvar_autoconnect = os.environ['SDSSDB_AUTOCONNECT'].lower() - if envvar_autoconnect == '0' or envvar_autoconnect == 'false': + if "SDSSDB_AUTOCONNECT" in os.environ: + envvar_autoconnect = os.environ["SDSSDB_AUTOCONNECT"].lower() + if envvar_autoconnect == "0" or envvar_autoconnect == "false": return False else: return sdssdb.autoconnect @@ -111,14 +114,13 @@ class DatabaseConnection(six.with_metaclass(abc.ABCMeta)): auto_reflect = True def __init__(self, dbname=None, profile=None, autoconnect=None, dbversion=None): - self.profile = None self._config = {} self.dbname = dbname if dbname else self.dbname self.dbversion = dbversion or self.dbversion if self.dbversion: - self.dbname = f'{self.dbname}_{self.dbversion}' + self.dbname = f"{self.dbname}_{self.dbversion}" self.set_profile(profile=profile, connect=False) @@ -129,8 +131,9 @@ def __init__(self, dbname=None, profile=None, autoconnect=None, dbversion=None): self.connect(dbname=self.dbname, silent_on_fail=True) def __repr__(self): - return '<{} (dbname={!r}, profile={!r}, connected={})>'.format( - self.__class__.__name__, self.dbname, self.profile, self.connected) + return "<{} (dbname={!r}, profile={!r}, connected={})>".format( + self.__class__.__name__, self.dbname, self.profile, self.connected + ) def set_profile(self, profile=None, connect=True, **params): """Sets the profile from the configuration file. @@ -156,31 +159,29 @@ def set_profile(self, profile=None, connect=True, **params): previous_profile = self.profile if profile is not None: - - assert profile in config, 'profile not found in configuration file.' + assert profile in config, "profile not found in configuration file." self.profile = profile self._config = config[profile].copy() else: - # Get hostname hostname = socket.getfqdn() # Initially set location to local. - self.profile = 'local' + self.profile = "local" self._config = config[self.profile].copy() # Tries to find a profile whose domain matches the hostname for profile in config: - if 'domain' in config[profile] and config[profile]['domain'] is not None: - if re.match(config[profile]['domain'], hostname): + if "domain" in config[profile] and config[profile]["domain"] is not None: + if re.match(config[profile]["domain"], hostname): self.profile = profile self._config = config[profile].copy() # If the profile host matches the current hostname set the # value to None to force using localhost to prevent cases # in which the loopback is not configured properly in PostgreSQL. - if hostname == self._config['host']: - self._config['host'] = None + if hostname == self._config["host"]: + self._config["host"] = None break self._config.update(params) @@ -229,13 +230,15 @@ def connect(self, dbname=None, silent_on_fail=False, **connection_params): """ if self.profile is None: - raise RuntimeError('the profile was not set when ' - 'DatabaseConnection was instantiated. Use ' - 'set_profile to set the profile in runtime.') + raise RuntimeError( + "the profile was not set when " + "DatabaseConnection was instantiated. Use " + "set_profile to set the profile in runtime." + ) # Gets the necessary configuration values from the profile db_configuration = {} - for item in ['user', 'host', 'port']: + for item in ["user", "host", "port"]: if item in connection_params: db_configuration[item] = connection_params[item] else: @@ -244,14 +247,16 @@ def connect(self, dbname=None, silent_on_fail=False, **connection_params): dbname = dbname or self.dbname if dbname is None: - raise RuntimeError('the database name was not set when ' - 'DatabaseConnection was instantiated. ' - 'To set it in runtime change the dbname ' - 'attribute.') + raise RuntimeError( + "the database name was not set when " + "DatabaseConnection was instantiated. " + "To set it in runtime change the dbname " + "attribute." + ) - return self.connect_from_parameters(dbname=dbname, - silent_on_fail=silent_on_fail, - **db_configuration) + return self.connect_from_parameters( + dbname=dbname, silent_on_fail=silent_on_fail, **db_configuration + ) def connect_from_parameters(self, dbname=None, **params): """Initialises the database from a dictionary of parameters. @@ -272,18 +277,20 @@ def connect_from_parameters(self, dbname=None, **params): """ # Make hostname an alias of host. - if 'hostname' in params: - if 'host' not in params: - params['host'] = params.pop('hostname') + if "hostname" in params: + if "host" not in params: + params["host"] = params.pop("hostname") else: - raise KeyError('cannot use hostname and host at the same time.') + raise KeyError("cannot use hostname and host at the same time.") dbname = dbname or self.dbname if dbname is None: - raise RuntimeError('the database name was not set when ' - 'DatabaseConnection was instantiated. ' - 'To set it in runtime change the dbname ' - 'attribute.') + raise RuntimeError( + "the database name was not set when " + "DatabaseConnection was instantiated. " + "To set it in runtime change the dbname " + "attribute." + ) return self._conn(dbname, **params) @@ -309,7 +316,7 @@ def get_connection_uri(self): params = self.connection_params if not self.connected or params is None: - raise RuntimeError('The database is not connected.') + raise RuntimeError("The database is not connected.") return get_database_uri(self.dbname, **params) @@ -332,18 +339,19 @@ def become(self, user): """Change the connection to a certain user.""" if not self.connected: - raise RuntimeError('DB has not been initialised.') + raise RuntimeError("DB has not been initialised.") dsn_params = self.connection_params - dsn_params.pop('password', None) # Do not keep the password since it may change. + dsn_params.pop("password", None) # Do not keep the password since it may change. if dsn_params is None: - raise RuntimeError('cannot determine the DSN parameters. ' - 'The DB may be disconnected.') + raise RuntimeError( + "cannot determine the DSN parameters. " "The DB may be disconnected." + ) - dsn_params['user'] = user - if 'dbname' not in dsn_params: - dsn_params['dbname'] = self.dbname + dsn_params["user"] = user + if "dbname" not in dsn_params: + dsn_params["dbname"] = self.dbname self.connect_from_parameters(**dsn_params) @@ -354,12 +362,13 @@ def become_admin(self, admin=None): """ - assert self.profile is not None, \ - 'this connection was not initialised from a profile. Try using become().' + assert ( + self.profile is not None + ), "this connection was not initialised from a profile. Try using become()." - assert 'admin' in self._config, 'admin user not defined in profile' + assert "admin" in self._config, "admin user not defined in profile" - self.become(admin or self._config['admin']) + self.become(admin or self._config["admin"]) def become_user(self, user=None): """Becomes the read-only user. @@ -368,24 +377,25 @@ def become_user(self, user=None): """ - assert self.profile is not None, \ - 'this connection was not initialised from a profile. Try using become().' + assert ( + self.profile is not None + ), "this connection was not initialised from a profile. Try using become()." if user is None: - user = self._config['user'] if 'user' in self._config else None + user = self._config["user"] if "user" in self._config else None self.become(user) def change_version(self, dbversion=None): - ''' Change database version and attempt to reconnect + """Change database version and attempt to reconnect Parameters: dbversion (str): A database version - ''' + """ self.dbversion = dbversion - dbname, *dbver = self.dbname.split('_') - self.dbname = f'{dbname}_{self.dbversion}' if dbversion else dbname + dbname, *dbver = self.dbname.split("_") + self.dbname = f"{dbname}_{self.dbversion}" if dbversion else dbname self.connect(dbname=self.dbname, silent_on_fail=True) def post_connect(self): @@ -406,15 +416,12 @@ class PeeweeDatabaseConnection(DatabaseConnection, PostgresqlDatabase): """ def __init__(self, *args, **kwargs): - self.models = {} self.introspector = {} self._metadata = {} - autorollback = kwargs.pop('autorollback', True) - - PostgresqlDatabase.__init__(self, None, autorollback=autorollback) + PostgresqlDatabase.__init__(self, None) DatabaseConnection.__init__(self, *args, **kwargs) @property @@ -435,15 +442,15 @@ def connection_params(self): def _conn(self, dbname, silent_on_fail=False, **params): """Connects to the DB and tests the connection.""" - if 'password' not in params: - pgpass_params = {key: value - for key, value in params.copy().items() - if value is not None} + if "password" not in params: + pgpass_params = { + key: value for key, value in params.copy().items() if value is not None + } try: - params['password'] = pgpasslib.getpass(dbname=dbname, **pgpass_params) + params["password"] = pgpasslib.getpass(dbname=dbname, **pgpass_params) except pgpasslib.FileNotFound: - params['password'] = None + params["password"] = None PostgresqlDatabase.init(self, dbname, **params) self._metadata = {} @@ -453,14 +460,14 @@ def _conn(self, dbname, silent_on_fail=False, **params): self.dbname = dbname except OperationalError as ee: if not silent_on_fail: - log.warning(f'failed connecting to database {self.database!r}: {ee}') + log.warning(f"failed connecting to database {self.database!r}: {ee}") PostgresqlDatabase.init(self, None) if self.is_connection_usable() and self.auto_reflect: with self.atomic(): for model in self.models.values(): - if getattr(model._meta, 'use_reflection', False): - if hasattr(model, 'reflect'): + if getattr(model._meta, "use_reflection", False): + if hasattr(model, "reflect"): model.reflect() if self.connected: @@ -498,18 +505,17 @@ def get_model(self, table_name, schema=None): def get_introspector(self, schema=None): """Gets a Peewee database :class:`peewee:Introspector`.""" - schema_key = schema or '' + schema_key = schema or "" if schema_key not in self.introspector: - self.introspector[schema_key] = Introspector.from_database( - self, schema=schema) + self.introspector[schema_key] = Introspector.from_database(self, schema=schema) return self.introspector[schema_key] def get_fields(self, table_name, schema=None, cache=True): """Returns a list of Peewee fields for a table.""" - schema = schema or 'public' + schema = schema or "public" if schema not in self._metadata or not cache: self._metadata[schema] = get_database_columns(self, schema=schema) @@ -519,21 +525,21 @@ def get_fields(self, table_name, schema=None, cache=True): table_metadata = self._metadata[schema][table_name] - pk = table_metadata['pk'] + pk = table_metadata["pk"] composite_key = pk is not None and len(pk) > 1 - columns = table_metadata['columns'] + columns = table_metadata["columns"] fields = [] for col_name, field_type, array_type, nullable in columns: + is_pk = True if (pk is not None and not composite_key and pk[0] == col_name) else False - is_pk = True if (pk is not None and not composite_key and - pk[0] == col_name) else False - - params = {'column_name': col_name, - 'null': nullable, - 'primary_key': is_pk, - 'unique': is_pk} + params = { + "column_name": col_name, + "null": nullable, + "primary_key": is_pk, + "unique": is_pk, + } if array_type: field = ArrayField(array_type, **params) @@ -549,7 +555,7 @@ def get_fields(self, table_name, schema=None, cache=True): def get_primary_keys(self, table_name, schema=None, cache=True): """Returns the primary keys for a table.""" - schema = schema or 'public' + schema = schema or "public" if schema not in self._metadata or not cache: self._metadata[schema] = get_database_columns(self, schema=schema) @@ -557,11 +563,11 @@ def get_primary_keys(self, table_name, schema=None, cache=True): if table_name not in self._metadata[schema]: return [] else: - return self._metadata[schema][table_name]['pk'] or [] + return self._metadata[schema][table_name]["pk"] or [] class SQLADatabaseConnection(DatabaseConnection): - ''' SQLAlchemy database connection implementation ''' + """SQLAlchemy database connection implementation""" engine = None bases = [] @@ -569,7 +575,6 @@ class SQLADatabaseConnection(DatabaseConnection): metadata = None def __init__(self, *args, **kwargs): - #: Reports whether the connection is active. self.connected = False @@ -583,7 +588,7 @@ def connection_params(self): return self._connect_params def _get_password(self, **params): - ''' Get a db password from a pgpass file + """Get a db password from a pgpass file Parameters: params (dict): @@ -592,19 +597,23 @@ def _get_password(self, **params): Returns: The database password for a given set of connection parameters - ''' + """ - password = params.get('password', None) + password = params.get("password", None) if not password: try: - password = pgpasslib.getpass(params['host'], params['port'], - params['database'], params['username']) + password = pgpasslib.getpass( + params["host"], + params["port"], + params["database"], + params["username"], + ) except KeyError: - raise RuntimeError('ERROR: invalid server configuration') + raise RuntimeError("ERROR: invalid server configuration") return password def _make_connection_string(self, dbname, **params): - ''' Build a db connection string + """Build a db connection string Parameters: dbname (str): @@ -615,33 +624,32 @@ def _make_connection_string(self, dbname, **params): Returns: A database connection string - ''' + """ db_params = params.copy() - db_params['drivername'] = 'postgresql+psycopg2' - db_params['database'] = dbname - db_params['username'] = db_params.pop('user', None) - db_params['host'] = db_params.pop('host', 'localhost') - db_params['port'] = db_params.pop('port', 5432) - if db_params['username']: - db_params['password'] = self._get_password(**db_params) + db_params["drivername"] = "postgresql+psycopg2" + db_params["database"] = dbname + db_params["username"] = db_params.pop("user", None) + db_params["host"] = db_params.pop("host", "localhost") + db_params["port"] = db_params.pop("port", 5432) + if db_params["username"]: + db_params["password"] = self._get_password(**db_params) db_connection_string = url.URL(**db_params) self._connect_params = params return db_connection_string def _conn(self, dbname, silent_on_fail=False, **params): - '''Connects to the DB and tests the connection.''' + """Connects to the DB and tests the connection.""" # get connection string db_connection_string = self._make_connection_string(dbname, **params) try: - self.create_engine(db_connection_string, echo=False, - pool_size=10, pool_recycle=1800) + self.create_engine(db_connection_string, echo=False, pool_size=10, pool_recycle=1800) self.engine.connect() except OpError: if not silent_on_fail: - log.warning('Failed to connect to database {0}'.format(dbname)) + log.warning("Failed to connect to database {0}".format(dbname)) self.engine.dispose() self.engine = None self.connected = False @@ -658,7 +666,7 @@ def _conn(self, dbname, silent_on_fail=False, **params): return self.connected def reset_engine(self): - ''' Reset the engine, metadata, and session ''' + """Reset the engine, metadata, and session""" self.bases = [] if self.engine: @@ -668,27 +676,37 @@ def reset_engine(self): self.Session.close() self.Session = None - def create_engine(self, db_connection_string=None, echo=False, pool_size=10, - pool_recycle=1800, expire_on_commit=True): - ''' Create a new database engine + def create_engine( + self, + db_connection_string=None, + echo=False, + pool_size=10, + pool_recycle=1800, + expire_on_commit=True, + ): + """Create a new database engine Resets and creates a new sqlalchemy database engine. Also creates and binds engine metadata and a new scoped session. - ''' + """ self.reset_engine() if not db_connection_string: dbname = self.dbname or self.DATABASE_NAME - db_connection_string = self._make_connection_string(dbname, - **self.connection_params) - - self.engine = create_engine(db_connection_string, echo=echo, pool_size=pool_size, - pool_recycle=pool_recycle) + db_connection_string = self._make_connection_string(dbname, **self.connection_params) + + self.engine = create_engine( + db_connection_string, + echo=echo, + pool_size=pool_size, + pool_recycle=pool_recycle, + ) self.metadata = MetaData(bind=self.engine) - self.Session = scoped_session(sessionmaker(bind=self.engine, autocommit=True, - expire_on_commit=expire_on_commit)) + self.Session = scoped_session( + sessionmaker(bind=self.engine, autocommit=True, expire_on_commit=expire_on_commit) + ) def add_base(self, base, prepare=True): """Binds a base to this connection.""" @@ -717,7 +735,7 @@ def prepare_bases(self, base=None): # If the base has an attribute _relations that's the function # to call to set up the relationships once the engine has been # bound to the base. - if hasattr(base, '_relations'): + if hasattr(base, "_relations"): if isinstance(base._relations, str): module = importlib.import_module(base.__module__) relations_func = getattr(module, base._relations) diff --git a/python/sdssdb/peewee/lvmdb/gortdb.py b/python/sdssdb/peewee/lvmdb/gortdb.py index 5c27d61b..8c685b07 100644 --- a/python/sdssdb/peewee/lvmdb/gortdb.py +++ b/python/sdssdb/peewee/lvmdb/gortdb.py @@ -6,9 +6,12 @@ # @Filename: gortdb.py # @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) +# fmt: on + from __future__ import annotations import datetime + from peewee import ( AutoField, BigIntegerField, @@ -18,13 +21,12 @@ FloatField, ForeignKeyField, IntegerField, - TextField + TextField, ) from playhouse.postgres_ext import JSONField from .. import BaseModel from . import database -from .lvmopsdb import Exposure as LVMOpsDBExposure class GortDBBase(BaseModel): @@ -43,7 +45,7 @@ class Overhead(GortDBBase): duration = FloatField() class Meta: - table_name = 'overhead' + table_name = "overhead" class Event(GortDBBase): @@ -54,7 +56,7 @@ class Event(GortDBBase): payload = JSONField() class Meta: - table_name = 'event' + table_name = "event" class Notification(GortDBBase): @@ -68,7 +70,7 @@ class Notification(GortDBBase): slack = BooleanField() class Meta: - table_name = 'notification' + table_name = "notification" class Exposure(GortDBBase): @@ -84,7 +86,7 @@ class Exposure(GortDBBase): header = JSONField() class Meta: - table_name = 'exposure' + table_name = "exposure" class NightLog(GortDBBase): @@ -93,15 +95,15 @@ class NightLog(GortDBBase): sent = BooleanField() class Meta: - table_name = 'night_log' + table_name = "night_log" class NightLogComment(GortDBBase): pk = AutoField(primary_key=True) - night_log = ForeignKeyField(model=NightLog, - column_name='night_log_pk', - backref='comments') + night_log = ForeignKeyField( + model=NightLog, column_name="night_log_pk", backref="comments" + ) comment = TextField() class Meta: - table_name = 'night_log_comment' + table_name = "night_log_comment" diff --git a/python/sdssdb/peewee/sdss5db/astradb.py b/python/sdssdb/peewee/sdss5db/astradb.py index ff9c1031..dc88e20b 100644 --- a/python/sdssdb/peewee/sdss5db/astradb.py +++ b/python/sdssdb/peewee/sdss5db/astradb.py @@ -7,7 +7,7 @@ # Database: sdss5db # Peewee version: 3.17.1 -# flake8: noqa=E501,E741 +# flake8: noqa: E501,E741 from peewee import (AutoField, BigIntegerField, BigBitField, BitField, BooleanField, DateTimeField, DoubleField, FloatField, ForeignKeyField, diff --git a/python/sdssdb/sqlalchemy/sdss5db/apogee_drpdb.py b/python/sdssdb/sqlalchemy/sdss5db/apogee_drpdb.py index f86a1317..95bef970 100644 --- a/python/sdssdb/sqlalchemy/sdss5db/apogee_drpdb.py +++ b/python/sdssdb/sqlalchemy/sdss5db/apogee_drpdb.py @@ -3,7 +3,7 @@ # # this file was autogenerated with sqlacodegen and then adapted for sdssdb -# flake8: noqa=E501,E741 +# flake8: noqa: E501,E741 from sqlalchemy import (ARRAY, BigInteger, Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, SmallInteger, Text, UniqueConstraint, text) diff --git a/python/sdssdb/sqlalchemy/sdss5db/astradb.py b/python/sdssdb/sqlalchemy/sdss5db/astradb.py index 4a79ed30..ccf50bb1 100644 --- a/python/sdssdb/sqlalchemy/sdss5db/astradb.py +++ b/python/sdssdb/sqlalchemy/sdss5db/astradb.py @@ -1,7 +1,7 @@ # coding: utf-8 # this file was autogenerated with sqlacodegen and then adapted for sdssdb -# flake8: noqa=E501,E741 +# flake8: noqa: E501,E741 from sqlalchemy.ext.declarative import AbstractConcreteBase, declared_attr from sqlalchemy import ARRAY, BigInteger, Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, LargeBinary, Text, text diff --git a/python/sdssdb/sqlalchemy/sdss5db/boss_drp.py b/python/sdssdb/sqlalchemy/sdss5db/boss_drp.py index da70bbed..a4c8ddcc 100644 --- a/python/sdssdb/sqlalchemy/sdss5db/boss_drp.py +++ b/python/sdssdb/sqlalchemy/sdss5db/boss_drp.py @@ -3,7 +3,7 @@ # # this file was autogenerated with sqlacodegen and then adapted for sdssdb -# flake8: noqa=E501,E741 +# flake8: noqa: E501,E741 from sqlalchemy import (ARRAY, BigInteger, Boolean, Column, DateTime, Float, ForeignKey, Integer, LargeBinary, SmallInteger, String, text) diff --git a/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py b/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py index 1e4d8913..4168ff82 100644 --- a/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py +++ b/python/sdssdb/sqlalchemy/sdss5db/catalogdb.py @@ -3,7 +3,7 @@ # # this file was autogenerated with sqlacodegen and then adapted for sdssdb -# flake8: noqa=E501,E741 +# flake8: noqa: E501,E741 from sqlalchemy import ( ARRAY, diff --git a/python/sdssdb/sqlalchemy/sdss5db/opsdb.py b/python/sdssdb/sqlalchemy/sdss5db/opsdb.py index 2b2b5162..5114bc83 100644 --- a/python/sdssdb/sqlalchemy/sdss5db/opsdb.py +++ b/python/sdssdb/sqlalchemy/sdss5db/opsdb.py @@ -3,10 +3,9 @@ # # this file was autogenerated with sqlacodegen and then adapted for sdssdb -# flake8: noqa=E501 +# flake8: noqa: E501 import os -from ast import In from sqlalchemy import ( ARRAY, diff --git a/python/sdssdb/sqlalchemy/sdss5db/targetdb.py b/python/sdssdb/sqlalchemy/sdss5db/targetdb.py index 5ea5dec2..944a4bd8 100644 --- a/python/sdssdb/sqlalchemy/sdss5db/targetdb.py +++ b/python/sdssdb/sqlalchemy/sdss5db/targetdb.py @@ -3,9 +3,7 @@ # # this file was autogenerated with sqlacodegen and then adapted for sdssdb -# flake8: noqa=E501 - -from cgi import print_form +# flake8: noqa: E501 from sqlalchemy import ( ARRAY, diff --git a/python/sdssdb/sqlalchemy/sdss5db/vizdb.py b/python/sdssdb/sqlalchemy/sdss5db/vizdb.py index 195c43d5..eb6b2c4a 100644 --- a/python/sdssdb/sqlalchemy/sdss5db/vizdb.py +++ b/python/sdssdb/sqlalchemy/sdss5db/vizdb.py @@ -3,7 +3,7 @@ # # this file was autogenerated with sqlacodegen and then adapted for sdssdb -# flake8: noqa=E501,E741 +# flake8: noqa: E501,E741 from sqlalchemy.ext.declarative import AbstractConcreteBase, declared_attr diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e2eca799..00000000 --- a/setup.cfg +++ /dev/null @@ -1,120 +0,0 @@ -[metadata] -name = sdssdb -version = 0.12.5a0 -author = José Sánchez-Gallego -author_email = gallegoj@uw.edu -description = SDSS product for database management -url = https://github.com/sdss/sdssdb -project_urls = - Repository = https://github.com/sdss/sdssdb - Documentation = https://sdssdb.readthedocs.org -long_description = file: README.md -long_description_content_type = text/markdown -keywords = astronomy, software, database -license = BSD 3-Clause License -license_file = LICENSE.md -classifiers = - Development Status :: 4 - Beta - Intended Audience :: Science/Research - Natural Language :: English - Operating System :: OS Independent - Programming Language :: Python - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Topic :: Documentation :: Sphinx - Topic :: Software Development :: Libraries :: Python Modules - -[options] -zip_safe = False -python_requires = >=3.6 -packages = find: -package_dir = - = python -install_requires = - pyyaml>=5.1 - pygments - pgpasslib>=1.1.0 - psycopg2-binary>=2.7.7 - six>=1.12.0 - peewee>=3.17.3 - sqlalchemy>=1.3.6 - sdsstools>=1.7.0; python_version >= "3.8" - sdsstools>=0.1.0,<1.0.0; python_version < "3.8" - numpy>=1.18.2 - h5py>=3.8.0 - -[options.packages.find] -where = - python -exclude = - *.tests - -[options.package_data] -sdssdb = - etc/* - -[options.extras_require] -all = - progressbar2>=3.46.1 - pydot>=1.4.1 - astropy>=4.0.0 - pandas>=1.0.0 - inflect>=4.1.0 -dev = - pytest>=5.2 - pytest-cov>=2.4.0 - pytest-sugar>=0.8.0 - ipython>=7.13.0 - ipdb>=0.13.2 - pytest-postgresql>=2.2.1 - factory_boy>=2.12.0 - pytest-factoryboy>=2.0.3 - astropy>=4.0.0 - pydot>=1.4.2 -docs = - Sphinx>=7.0.0 - sphinx_bootstrap_theme>=0.4.12 - releases>=2.0.0 - -[isort] -line_length = 79 -sections = - FUTURE - STDLIB - THIRDPARTY - SQLA - PEEWEE - SDSS - FIRSTPARTY - LOCALFOLDER -default_section = THIRDPARTY -known_first_party = - sdssdb -known_sqla = - sqlalchemy -known_peewee = - peewee - playhouse -known_sdss = - sdsstools -balanced_wrapping = true -include_trailing_comma = false -lines_after_imports = 2 -use_parentheses = true - -[flake8] -ignore = - H101 - E722 - W504 - W505 - E116 - E114 -max-line-length = 99 -exclude = - python/sdssdb/sqlalchemy/operationsdb/tools/ - python/sdssdb/sqlalchemy/archive - python/sdssdb/sqlalchemy/mangadb - -[coverage:run] -branch = true diff --git a/setup.py b/setup.py deleted file mode 100644 index 578f6d67..00000000 --- a/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -# encoding: utf-8 -# -# setup.py -# - -from setuptools import setup - - -setup() diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..5a58977b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# @Author: José Sánchez-Gallego (gallegoj@uw.edu) +# @Date: 2024-09-02 +# @Filename: __init__.py +# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause) + +from __future__ import annotations + +import warnings + +from sqlalchemy.exc import MovedIn20Warning + + +warnings.filterwarnings("ignore", category=MovedIn20Warning) diff --git a/tests/conftest.py b/tests/conftest.py index 3dc60736..9aa2778a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,52 +3,66 @@ # conftest.py # -from __future__ import print_function, division, absolute_import, unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals -import re -import pytest import importlib import inspect -from .sqladbs import prepare_testdb as sqla_prepdb +import re + +import pytest +from pytest_postgresql.janitor import DatabaseJanitor + from .pwdbs import prepare_testdb as pw_prepdb -from pytest_postgresql.factories import DatabaseJanitor +from .sqladbs import prepare_testdb as sqla_prepdb def pytest_addoption(parser): - """ Add new options to the pytest command-line """ + """Add new options to the pytest command-line""" # only run peewee tests - parser.addoption('--peewee', action='store_true', default=False, - help='Only run tests for peewee dbs') + parser.addoption( + "--peewee", + action="store_true", + default=False, + help="Only run tests for peewee dbs", + ) # only run sqla tests - parser.addoption('--sqla', action='store_true', default=False, - help='Only run tests for sqlalchemy dbs') + parser.addoption( + "--sqla", + action="store_true", + default=False, + help="Only run tests for sqlalchemy dbs", + ) # persist the sqla session and peewee transaction - parser.addoption('--persist-sessions', action='store_true', default=False, - help='Switch session and transaction fixtures to module scope') + parser.addoption( + "--persist-sessions", + action="store_true", + default=False, + help="Switch session and transaction fixtures to module scope", + ) def pytest_ignore_collect(path, config): - ''' pytest hook to identify tests to be ignored during collection + """pytest hook to identify tests to be ignored during collection Looks through all test_xxx.py files in pwdbs and sqladbs and determines which ones have databases that fail to connect and ignores them. - ''' + """ only_peewee = config.getoption("--peewee", None) only_sqla = config.getoption("--sqla", None) - assert not all([only_peewee, only_sqla]), 'both --peewee and --sqla options cannot be set' + assert not all([only_peewee, only_sqla]), "both --peewee and --sqla options cannot be set" if only_peewee: - if 'sqladbs' in str(path): + if "sqladbs" in str(path): return True if only_sqla: - if 'pwdbs' in str(path): + if "pwdbs" in str(path): return True # identify and ignore test modules where no local # database is set up for those tests - if re.search('test_[a-z]+.py', str(path)): + if re.search("test_[a-z]+.py", str(path)): # get module name modname = inspect.getmodulename(path) # find and load the underlying module @@ -57,38 +71,38 @@ def pytest_ignore_collect(path, config): # execute load spec.loader.exec_module(foo) # get the database from the module - db = getattr(foo, 'database', None) + db = getattr(foo, "database", None) # check if db is connected - if db and db.connected is False and db.dbname != 'test': + if db and db.connected is False and db.dbname != "test": return True -@pytest.fixture(scope='module', autouse=True) +@pytest.fixture(scope="module", autouse=True) def skipdb(database): - ''' fixture to skip database tests if the db does not exist ''' + """fixture to skip database tests if the db does not exist""" if database.connected is False: - pytest.skip(f'no {database.dbname} found') + pytest.skip(f"no {database.dbname} found") database = None -@pytest.fixture(scope='module') -def dropdb(): - janitor = DatabaseJanitor('postgres', 'localhost', 5432, 'test', '11.4') - janitor.drop() - - -@pytest.fixture(scope='module') -def database(dropdb, request): - ''' Module fixture to initialize a real database or a test postgresql database ''' - if hasattr(request, 'param'): +@pytest.fixture(scope="module") +def database(request): + """Module fixture to initialize a real database or a test postgresql database""" + if hasattr(request, "param"): # yield a real database yield request.param else: # check if request is coming from a sqla db or peewee db - issqla = 'sqladbs' in request.module.__name__ or 'sqlalchemy' in request.module.__name__ + issqla = "sqladbs" in request.module.__name__ or "sqlalchemy" in request.module.__name__ # initialize the test database # uses https://github.com/ClearcodeHQ/pytest-postgresql - janitor = DatabaseJanitor('postgres', 'localhost', 5432, 'test', '11.4') + janitor = DatabaseJanitor( + user="postgres", + host="localhost", + port=5432, + dbname="test", + version="11.4", + ) janitor.init() db = sqla_prepdb() if issqla else pw_prepdb() yield db @@ -97,8 +111,7 @@ def database(dropdb, request): def determine_scope(fixture_name, config): - ''' determine the scope of the session and transaction fixtures ''' + """determine the scope of the session and transaction fixtures""" if config.getoption("--persist-sessions", None): return "module" return "function" -