From 5a2631a44fed49999ea93e6b160b37efb0e568b7 Mon Sep 17 00:00:00 2001 From: alphasentaurii Date: Mon, 3 Jul 2023 15:15:01 -0400 Subject: [PATCH] increment version to 0.4.1, updated docs --- CHANGES.rst | 24 ++++++ docker/images/dashboard_image/.env-template | 2 +- docker/images/dashboard_image/Dockerfile | 2 +- .../images/dashboard_image/templates/aws/.env | 2 +- .../images/dashboard_image/templates/dev/.env | 2 +- .../dashboard_image/templates/latest/.env | 2 +- docs/source/builder/blueprints.rst | 12 +++ docs/source/datasets/beam.rst | 30 ++++++++ docs/source/datasets/hst_cal.rst | 3 - docs/source/datasets/hst_svm.rst | 3 - docs/source/datasets/k2_exo.rst | 3 - docs/source/index.rst | 16 +++- docs/source/logger/log.rst | 8 ++ docs/source/preprocessor/ingest.rst | 8 ++ docs/source/preprocessor/prep.rst | 8 ++ docs/source/skopes/hst/cal.rst | 7 ++ docs/source/skopes/hst/cal/predict.rst | 8 ++ docs/source/skopes/hst/cal/train.rst | 77 ++++++++++--------- setup.cfg | 2 +- 19 files changed, 165 insertions(+), 54 deletions(-) create mode 100644 docs/source/builder/blueprints.rst create mode 100644 docs/source/datasets/beam.rst delete mode 100644 docs/source/datasets/hst_cal.rst delete mode 100644 docs/source/datasets/hst_svm.rst delete mode 100644 docs/source/datasets/k2_exo.rst create mode 100644 docs/source/logger/log.rst create mode 100644 docs/source/preprocessor/ingest.rst create mode 100644 docs/source/preprocessor/prep.rst create mode 100644 docs/source/skopes/hst/cal/predict.rst diff --git a/CHANGES.rst b/CHANGES.rst index 29340cc..f5e2c75 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,27 @@ +0.4.1 (2023-07-03) +================== + +- bugfix set dataframe columns with bracket instead of curly bracket (resolves pandas>1.4 incompatibility) + +- remove pandas pinned version + +- improved log handling with spacekit/logger module + +- added predict script for hst cal skope + +- updated docker dashboard templates + +- enhancements for loading pretrained models + +- pytest configuration updates and new tests added + +- plugin for external test data + +- updated repo url badges + +- updated documentation + + 0.4.0 (2022-12-08) ================== diff --git a/docker/images/dashboard_image/.env-template b/docker/images/dashboard_image/.env-template index 05d2452..d50c98d 100644 --- a/docker/images/dashboard_image/.env-template +++ b/docker/images/dashboard_image/.env-template @@ -3,7 +3,7 @@ VERSION="test" TAG="dash-${APP}-${VERSION}" DOCKER_REPO="alphasentaurii/spacekit" DOCKER_IMAGE="${DOCKER_REPO}:${TAG}" -SPACEKIT_VERSION="0.4.0" +SPACEKIT_VERSION="0.4.1" SRC="git" # pkg, s3, git, file COLLECTION="calcloud" # bucketname, repo url, or local path DATASETS="2022-02-14,2021-11-04,2021-10-28" diff --git a/docker/images/dashboard_image/Dockerfile b/docker/images/dashboard_image/Dockerfile index c0dc8b8..9c56016 100644 --- a/docker/images/dashboard_image/Dockerfile +++ b/docker/images/dashboard_image/Dockerfile @@ -29,7 +29,7 @@ ENV SPACEKIT_DATA=/home/developer ENV TF_CPP_MIN_LOG_LEVEL=2 ARG PFX=archive ENV PFX=${PFX} -ARG SPACEKIT_VERSION=0.4.0 +ARG SPACEKIT_VERSION=0.4.1 ENV SPACEKIT_VERSION=${SPACEKIT_VERSION} ARG SRC ARG COLLECTION diff --git a/docker/images/dashboard_image/templates/aws/.env b/docker/images/dashboard_image/templates/aws/.env index ae06103..ce750b9 100644 --- a/docker/images/dashboard_image/templates/aws/.env +++ b/docker/images/dashboard_image/templates/aws/.env @@ -3,7 +3,7 @@ VERSION="aws" TAG="dash-${APP}-${VERSION}" DOCKER_REPO="alphasentaurii/spacekit" DOCKER_IMAGE="${DOCKER_REPO}:${TAG}" -SPACEKIT_VERSION="0.4.0" +SPACEKIT_VERSION="0.4.1" SRC="s3" # pkg, s3, git, file COLLECTION="" # bucketname, repo url, or local path DATASETS="2022-02-14-1644848448,2021-11-04-1636048291,2021-10-28-1635457222" diff --git a/docker/images/dashboard_image/templates/dev/.env b/docker/images/dashboard_image/templates/dev/.env index 2079e7a..ba4c911 100644 --- a/docker/images/dashboard_image/templates/dev/.env +++ b/docker/images/dashboard_image/templates/dev/.env @@ -3,7 +3,7 @@ VERSION="dev" TAG="dash-${APP}-${VERSION}" DOCKER_REPO="alphasentaurii/spacekit" DOCKER_IMAGE="${DOCKER_REPO}:${TAG}" -SPACEKIT_VERSION="0.4.0" +SPACEKIT_VERSION="0.4.1" SRC="git" # pkg, s3, git, file COLLECTION="calcloud" # bucketname, repo url, or local path DATASETS="2022-02-14,2021-11-04,2021-10-28" diff --git a/docker/images/dashboard_image/templates/latest/.env b/docker/images/dashboard_image/templates/latest/.env index 82d5cff..b1befaf 100644 --- a/docker/images/dashboard_image/templates/latest/.env +++ b/docker/images/dashboard_image/templates/latest/.env @@ -3,7 +3,7 @@ VERSION="latest" TAG="dash-${APP}-${VERSION}" DOCKER_REPO="alphasentaurii/spacekit" DOCKER_IMAGE="${DOCKER_REPO}:${TAG}" -SPACEKIT_VERSION="0.4.0" +SPACEKIT_VERSION="0.4.1" SRC="git" # pkg, s3, git, file COLLECTION="calcloud" # bucketname, repo url, or local path DATASETS="2022-02-14,2021-11-04,2021-10-28" diff --git a/docs/source/builder/blueprints.rst b/docs/source/builder/blueprints.rst new file mode 100644 index 0000000..5b427b0 --- /dev/null +++ b/docs/source/builder/blueprints.rst @@ -0,0 +1,12 @@ +.. _blueprint: + +*************************** +spacekit.builder.blueprints +*************************** + +.. currentmodule:: spacekit.builder.blueprints + +.. autoclass:: Blueprint + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/datasets/beam.rst b/docs/source/datasets/beam.rst new file mode 100644 index 0000000..4a5502c --- /dev/null +++ b/docs/source/datasets/beam.rst @@ -0,0 +1,30 @@ +.. _beam: + +********************** +spacekit.datasets.beam +********************** + +.. currentmodule:: spacekit.datasets.beam + +.. autofunction:: download + +Retrieve dataset archive (.zip) files from the web, an s3 bucket, or local disk. +This script is primarily intended as a retrieval and extraction step before launching spacekit.dashboard +For more customized control of dataset retrieval (such as your own), use the spacekit.extractor.scrape module. + +Examples: +"datasets": if set, chooses specific archive dataset filenames to retrieve and extract + +src: "git" - Fetch and extract from one of the spacekit data archives: +archive: name of collection (see ``spacekit.datasets.meta``) +download(scrape="git:calcloud") +download(scrape="git:svm") + +src: "file" - Fetch and extract from path on local disk +archive: path +download(scrape="file:another/path/to/data) + +src: "s3" - Fetch and extract from an S3 bucket on AWS +archive: bucketname +"datasets" - string of S3 archive file basenames separated by comma (without the .zip or .tgz suffix) +download(scrape="s3:mybucketname", "2021-11-04-1636048291,2021-10-28-1635457222,2021-08-22-1629663047") \ No newline at end of file diff --git a/docs/source/datasets/hst_cal.rst b/docs/source/datasets/hst_cal.rst deleted file mode 100644 index 4b2ae9d..0000000 --- a/docs/source/datasets/hst_cal.rst +++ /dev/null @@ -1,3 +0,0 @@ -************************* -spacekit.datasets.hst_cal -************************* \ No newline at end of file diff --git a/docs/source/datasets/hst_svm.rst b/docs/source/datasets/hst_svm.rst deleted file mode 100644 index a2fe0b9..0000000 --- a/docs/source/datasets/hst_svm.rst +++ /dev/null @@ -1,3 +0,0 @@ -************************* -spacekit.datasets.hst_svm -************************* \ No newline at end of file diff --git a/docs/source/datasets/k2_exo.rst b/docs/source/datasets/k2_exo.rst deleted file mode 100644 index a7e4f1e..0000000 --- a/docs/source/datasets/k2_exo.rst +++ /dev/null @@ -1,3 +0,0 @@ -************************ -spacekit.datasets.k2_exo -************************ \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 7b6eb3a..29513bb 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -28,6 +28,7 @@ Builder :maxdepth: 1 architect + blueprints Dashboard @@ -46,9 +47,7 @@ Datasets .. toctree:: :maxdepth: 1 - hst_cal - hst_svm - k2_exo + beam Extractor @@ -72,6 +71,15 @@ Generator draw +Logger +------ + +.. toctree:: + :maxdepth: 1 + + log + + Preprocessor ------------ @@ -79,6 +87,8 @@ Preprocessor :maxdepth: 1 encode + ingest + prep scrub transform diff --git a/docs/source/logger/log.rst b/docs/source/logger/log.rst new file mode 100644 index 0000000..fa5535a --- /dev/null +++ b/docs/source/logger/log.rst @@ -0,0 +1,8 @@ +.. _log: + +******************* +spacekit.logger.log +******************* + +.. automodule:: spacekit.logger.log + :members: \ No newline at end of file diff --git a/docs/source/preprocessor/ingest.rst b/docs/source/preprocessor/ingest.rst new file mode 100644 index 0000000..8b98963 --- /dev/null +++ b/docs/source/preprocessor/ingest.rst @@ -0,0 +1,8 @@ +.. _ingest: + +**************************** +spacekit.preprocessor.ingest +**************************** + +.. automodule:: spacekit.preprocessor.ingest + :members: \ No newline at end of file diff --git a/docs/source/preprocessor/prep.rst b/docs/source/preprocessor/prep.rst new file mode 100644 index 0000000..39c08c9 --- /dev/null +++ b/docs/source/preprocessor/prep.rst @@ -0,0 +1,8 @@ +.. _prep: + +**************************** +spacekit.preprocessor.prep +**************************** + +.. automodule:: spacekit.preprocessor.prep + :members: diff --git a/docs/source/skopes/hst/cal.rst b/docs/source/skopes/hst/cal.rst index a7fb594..47d5427 100644 --- a/docs/source/skopes/hst/cal.rst +++ b/docs/source/skopes/hst/cal.rst @@ -6,6 +6,13 @@ Calcloud Job Predict Data Dictionary ------------------------------------ +.. toctree:: + :maxdepth: 2 + + predict + train + + DETECTOR -------- diff --git a/docs/source/skopes/hst/cal/predict.rst b/docs/source/skopes/hst/cal/predict.rst new file mode 100644 index 0000000..9e9c6cb --- /dev/null +++ b/docs/source/skopes/hst/cal/predict.rst @@ -0,0 +1,8 @@ +******************************* +spacekit.skopes.hst.svm.predict +******************************* + +.. currentmodule:: spacekit.skopes.hst.cal.predict + +.. automodule:: spacekit.skopes.hst.cal.predict + :members: \ No newline at end of file diff --git a/docs/source/skopes/hst/cal/train.rst b/docs/source/skopes/hst/cal/train.rst index 441e287..ef923d6 100644 --- a/docs/source/skopes/hst/cal/train.rst +++ b/docs/source/skopes/hst/cal/train.rst @@ -33,42 +33,47 @@ python -m spacekit.skopes.hst.cal.train data/2021-11-04-1636048291 using this script produces a specific file output structure which can optionally be compressed and uploaded to s3. The dataset used for training is updated with model predictions and can optionally be stored in a dynamodb table. -|- data/2021-11-04-1636048291 - |- data - |- latest.csv - |- models - |- tx_data.json - |- mem_clf - |- {saved model binaries} - |- mem_reg - |- {saved model binaries} - |- wall_reg - |- {saved model binaries} - |- results - |- mem_bin {standard classifier filenames} - |- acc_loss - |- cmx - |- cmx_norm - |- fnfp - |- history - |- report - |- roc_auc - |- test_idx - |- y_onehot - |- y_pred - |- y_scores - |- memory {standard regressor filenames} - |- history - |- loss - |- predictions - |- residuals - |- test_idx - |- wallclock {standard regressor filenames} - |- history - |- loss - |- predictions - |- residuals - |- test_idx + +.. code-block:: python + + """ + |- data/2021-11-04-1636048291 + |- data + |- latest.csv + |- models + |- tx_data.json + |- mem_clf + |- {saved model binaries} + |- mem_reg + |- {saved model binaries} + |- wall_reg + |- {saved model binaries} + |- results + |- mem_bin {standard classifier filenames} + |- acc_loss + |- cmx + |- cmx_norm + |- fnfp + |- history + |- report + |- roc_auc + |- test_idx + |- y_onehot + |- y_pred + |- y_scores + |- memory {standard regressor filenames} + |- history + |- loss + |- predictions + |- residuals + |- test_idx + |- wallclock {standard regressor filenames} + |- history + |- loss + |- predictions + |- residuals + |- test_idx + """ examples: diff --git a/setup.cfg b/setup.cfg index f37262e..c46e8fd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = spacekit -version = 0.4.0 +version = 0.4.1 author = Ru Keïn author_email = rkein@stsci.edu license = MIT