Merge pull request #1 from mkstratos/mkstratos/system_updates

System updates

.git-blame-ignore-revs

@@ -1 +1,2 @@
-0e03e4395fae0aa0b6ab475feb054e53135f2352
+0e03e4395fae0aa0b6ab475feb054e53135f2352
+09fe85c8c46dceb03d9dfa4a0d6453e308c02712

.github/workflows/pre-commit.yml

@@ -0,0 +1,27 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Pre-Commit
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+
+  pre-commit:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Runs pre-commit
+        run: |
+          pip install pre-commit
+          pre-commit run -a

.github/workflows/python-package.yml

@@ -0,0 +1,34 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install pytest
+        python3 -m pip install -e .\[dev\]
+    - name: Test with pytest
+      run: |
+        pytest -vvv

.github/workflows/python-publish.yml

@@ -0,0 +1,39 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -2,6 +2,7 @@
 
 test/*_test
 test/*_test_*
+test/data
 
 #####=== Vim ===#####
 [._]*.s[a-w][a-z]
@@ -67,4 +68,3 @@ docs/_build/
 
 # PyBuilder
 target/
-

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-toml
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.3.7
+    hooks:
+      # Run the linter.
+      - id: ruff

LICENSE

@@ -25,4 +25,3 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-

README.md

@@ -20,6 +20,6 @@ Want to send us a private message?
 * github: @mkstratos
 * email: <a href="mailto:[email protected]">kelleherme [at] ornl.gov</a>
 
-**Joseph H. Kennedy** 
+**Joseph H. Kennedy**
 * github: @jhkennedy
 * email: <a href="mailto:[email protected]">kennedyjh [at] ornl.gov</a>

evv4esm/__main__.py

@@ -28,15 +28,16 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import argparse
 import os
 import sys
 import time
-import argparse
 
-import evv4esm
 import livvkit
 from livvkit.util import options
 
+import evv4esm
+
 
 def parse_args(args=None):
     parser = argparse.ArgumentParser(
@@ -110,27 +111,26 @@ def main(cl_args=None):
         cl_args = sys.argv[1:]
     args = parse_args(cl_args)
 
-    print("--------------------------------------------------------------------")
-    print("                   ______  __      __ __      __                    ")
-    print("                  |  ____| \ \    / / \ \    / /                    ")
-    print("                  | |__     \ \  / /   \ \  / /                     ")
-    print("                  |  __|     \ \/ /     \ \/ /                      ")
-    print("                  | |____     \  /       \  /                       ")
-    print("                  |______|     \/         \/                        ")
-    print("                                                                    ")
-    print("    Extended Verification and Validation for Earth System Models    ")
-    print("--------------------------------------------------------------------")
+    print(r"--------------------------------------------------------------------")
+    print(r"                   ______  __      __ __      __                    ")
+    print(r"                  |  ____| \ \    / / \ \    / /                    ")
+    print(r"                  | |__     \ \  / /   \ \  / /                     ")
+    print(r"                  |  __|     \ \/ /     \ \/ /                      ")
+    print(r"                  | |____     \  /       \  /                       ")
+    print(r"                  |______|     \/         \/                        ")
+    print(r"                                                                    ")
+    print(r"    Extended Verification and Validation for Earth System Models    ")
+    print(r"--------------------------------------------------------------------")
     print("")
     print("  Current run: " + livvkit.timestamp)
     print("  User: " + livvkit.user)
     print("  OS Type: " + livvkit.os_type)
     print("  Machine: " + livvkit.machine)
     print("  " + livvkit.comment)
 
+    from livvkit import elements, scheduler
     from livvkit.components import validation
-    from livvkit import scheduler
     from livvkit.util import functions
-    from livvkit import elements
 
     livvkit.pool_size = args.pool_size
     if args.extensions:

evv4esm/ensembles/e3sm.py

@@ -30,17 +30,15 @@
 
 """E3SM specific ensemble functions."""
 
-import six
-
+import glob
 import os
 import re
-import glob
-
 from collections import OrderedDict
 from functools import partial
 
 import numpy as np
 import pandas as pd
+import six
 from netCDF4 import Dataset
 
 

evv4esm/ensembles/tools.py

@@ -30,12 +30,12 @@
 
 """General tools for working with ensembles."""
 import os
+
+import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-import matplotlib.pyplot as plt
-from matplotlib.gridspec import GridSpec
 
-from evv4esm import pf_color_picker, light_pf_color_picker
+from evv4esm import light_pf_color_picker, pf_color_picker
 
 
 def monthly_to_annual_avg(var_data, cal="ignore"):

evv4esm/extensions/ks.bib

@@ -25,4 +25,3 @@ @article{MAHAJAN2019
 eprint = {https://doi.org/10.1177/1094342019837341},
 abstract = {We present a methodology for solution reproducibility for the Energy Exascale Earth System Model during its ongoing software infrastructure development to prepare for exascale computers. The nonlinear chaotic nature of climate system simulations precludes traditional model verification approaches since machine precision differences—resulting from code refactoring, changes in software environment, and so on—grow exponentially to a different weather state. Here, we leverage the nature of climate as a statistical description of the atmosphere in order to establish model reproducibility. We evaluate the degree to which two-sample equality of distribution tests can confidently detect the change in climate from minor tuning parameter changes on model output variables in order to establish the level of difference that indicates a new climate. To apply this (baselined test), we target a section of the model’s development cycle wherein no intentional science changes have been applied to its source code. We compare an ensemble of short simulations that were conducted using a verified model configuration against a new ensemble with the same configuration but with the latest software infrastructure (Common Infrastructure for Modeling the Earth, CIME5.0), compiler versions, and software libraries. We also compare these against ensemble simulations conducted using the original version of the software infrastructure (CIME4.0) of the earlier model configuration, but with the latest compilers and software libraries, to test the impact of new compilers and libraries in isolation from additional software infrastructure. The two-sample equality of distribution tests indicates that these ensembles indeed represent the same climate.}
 }
-

evv4esm/extensions/ks_vars.json

@@ -157,4 +157,4 @@
               "soa_a1_SRF",
               "soa_a2_2",
               "soa_a2_SRF"]
-}
+}

evv4esm/extensions/kso.py

@@ -52,16 +52,17 @@
 import livvkit
 import numpy as np
 import pandas as pd
-from evv4esm import EVVException, human_color_names
-from evv4esm.ensembles import e3sm
-from evv4esm.ensembles.tools import prob_plot
-from evv4esm.utils import bib2html
 from livvkit import elements as el
 from livvkit.util import functions as fn
 from livvkit.util.LIVVDict import LIVVDict
 from scipy import stats
 from statsmodels.stats import multitest as smm
 
+from evv4esm import EVVException, human_color_names
+from evv4esm.ensembles import e3sm
+from evv4esm.ensembles.tools import prob_plot
+from evv4esm.utils import bib2html
+
 
 def variable_set(name):
     var_sets = fn.read_json(os.path.join(os.path.dirname(__file__), "ocean_vars.json"))

evv4esm/extensions/ocean_vars.json

@@ -6,4 +6,4 @@
         "velocityMeridional{}",
         "velocityZonal{}"
     ]
-}
+}

evv4esm/extensions/pg.bib

@@ -3,4 +3,4 @@ @article{SINGH2020
 title = {A verification strategy for atmospheric model codes using initial condition perturbations},
 journal = {Journal of Geophysical Research: Atmospheres},
 year = {In prep.}
-}
+}

evv4esm/extensions/pg.py

@@ -36,31 +36,23 @@
 simulation for many initial conditions, with each initial condition subject to
 multiple perturbations.
 """
-import six
-
-import os
-import math
 import argparse
-
-# import logging
+import math
+import os
+from collections import OrderedDict
 from pathlib import Path
-
 from pprint import pprint
-from collections import OrderedDict
 
+import livvkit
+import matplotlib.patches as mpatches
+import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-
-import matplotlib.pyplot as plt
-import matplotlib.patches as mpatches
-
-from scipy import stats
-from netCDF4 import Dataset
-
-import livvkit
+import six
 from livvkit import elements as el
 from livvkit.util import functions as fn
-from livvkit.util.LIVVDict import LIVVDict
+from netCDF4 import Dataset
+from scipy import stats
 
 from evv4esm.utils import bib2html
 

evv4esm/extensions/tsc.bib

@@ -9,4 +9,4 @@ @Article{gmd-10-537-2017
 PAGES = {537--552},
 URL = {https://www.geosci-model-dev.net/10/537/2017/},
 DOI = {10.5194/gmd-10-537-2017}
-}
+}

evv4esm/extensions/tsc.py

@@ -52,24 +52,23 @@
 import os
 from collections import OrderedDict
 from itertools import groupby
-from pprint import pprint
 from pathlib import Path
+from pprint import pprint
 
+import livvkit
+import matplotlib.pyplot as plt
+import matplotlib.ticker as tkr
 import numpy as np
 import pandas as pd
-import matplotlib.ticker as tkr
-import matplotlib.pyplot as plt
-from scipy import stats
-from netCDF4 import Dataset
-
-import livvkit
 from livvkit import elements as el
 from livvkit.util import functions as fn
 from livvkit.util.LIVVDict import LIVVDict
+from netCDF4 import Dataset
+from scipy import stats
 
+from evv4esm import human_color_names, light_pf_color_picker, pf_color_picker
 from evv4esm.ensembles import e3sm
 from evv4esm.utils import bib2html
-from evv4esm import pf_color_picker, light_pf_color_picker, human_color_names
 
 
 def parse_args(args=None):