diff --git a/.cmake-format.py b/.cmake-format.py new file mode 100644 index 0000000000..e54ca58d3a --- /dev/null +++ b/.cmake-format.py @@ -0,0 +1,331 @@ +# ---------------------------------- +# Options affecting listfile parsing +# ---------------------------------- +with section("parse"): + + # Specify structure for custom cmake functions + additional_commands = { + 'add_prefix': {'pargs': {'nargs': 2}}, + 'add_suffix': {'pargs': {'nargs': 2}}, + 'append_static_suffix': {'pargs': {'nargs': 2}}, + 'examples2string': {'pargs': {'nargs': 2}}, + 'force_variable': {'pargs': {'nargs': 4}}, + 'list2string': {'pargs': {'nargs': 2}}, + 'sundials_add_benchmark': { 'kwargs': { 'BENCHMARK_ARGS': 1, + 'IDENTIFIER': 1, + 'NUM_CORES': 1, + 'TEST_RUNNER_ARGS': '+'}, + 'pargs': { 'flags': ['ENABLE_GPU'], + 'nargs': '3+'}}, + 'sundials_add_examples_ginkgo': { 'kwargs': {'BACKENDS': '+', 'TARGETS': '+'}, + 'pargs': { 'flags': ['UNIT_TEST'], + 'nargs': '1+'}}, + 'sundials_add_f2003_library': { 'kwargs': { 'COMPILE_DEFINITIONS': '+', + 'COMPILE_OPTIONS': '+', + 'INCLUDE_DIRECTORIES': '+', + 'LINK_LIBRARIES': '+', + 'OBJECT_LIBRARIES': '+', + 'OUTPUT_NAME': 1, + 'PROPERTIES': '+', + 'SOURCES': '+', + 'SOVERSION': 1, + 'VERSION': 1}, + 'pargs': {'flags': [], 'nargs': '1+'}}, + 'sundials_add_library': { 'kwargs': { 'COMPILE_DEFINITIONS': '+', + 'COMPILE_FEATURES': '+', + 'COMPILE_OPTIONS': '+', + 'HEADERS': '+', + 'INCLUDE_DIRECTORIES': '+', + 'INCLUDE_SUBDIR': 1, + 'LINK_LIBRARIES': '+', + 'OBJECT_LIBRARIES': '+', + 'OUTPUT_NAME': 1, + 'PROPERTIES': '+', + 'SOURCES': '+', + 'SOVERSION': 1, + 'VERSION': 1}, + 'pargs': { 'flags': [ 'STATIC_ONLY', + 'SHARED_ONLY', + 'OBJECT_LIB_ONLY'], + 'nargs': '1+'}}, + 'sundials_add_nvector_benchmark': { 'kwargs': { 'INSTALL_SUBDIR': '+', + 'LINK_LIBRARIES': '+', + 'SOURCES': '+', + 'SUNDIALS_TARGETS': '+'}, + 'pargs': {'flags': [], 'nargs': '1+'}}, + 'sundials_add_test': { 'kwargs': { 'ANSWER_DIR': 1, + 'ANSWER_FILE': 1, + 'EXAMPLE_TYPE': 1, + 'EXTRA_ARGS': '+', + 'FLOAT_PRECISION': 1, + 'INTEGER_PRECISION': 1, + 'LABELS': '+', + 'MPI_NPROCS': 1, + 'TEST_ARGS': '+'}, + 'pargs': {'flags': ['NODIFF'], 'nargs': '2+'}}, + 'sundials_add_test_install': { 'kwargs': {'EXECUTABLE': 1}, + 'pargs': {'flags': [], 'nargs': '2+'}}, + 'sundials_git_version': {'pargs': {'nargs': 0}}, + 'sundials_install_examples': { 'kwargs': { 'CMAKE_TEMPLATE': 1, + 'DESTINATION': 1, + 'EXAMPLES_DEPENDENCIES': '+', + 'EXTRA_FILES': '+', + 'EXTRA_INCLUDES': '+', + 'MAKE_TEMPLATE': 1, + 'OTHER_TARGETS': '+', + 'SOLVER_LIBRARY': 1, + 'SUNDIALS_COMPONENTS': '+', + 'SUNDIALS_TARGETS': '+', + 'TEST_INSTALL': 1}, + 'pargs': {'flags': [], 'nargs': '2+'}}, + 'sundials_install_examples_ginkgo': { 'kwargs': { 'CPU_EXAMPLES_VAR': '+', + 'CPU_GPU_EXAMPLES_VAR': '+', + 'DEPENDENCIES': '+', + 'DESTINATION': 1, + 'EXTRA_FILES': '+', + 'GPU_EXAMPLES_VAR': '+', + 'SUNDIALS_COMPONENTS': '+', + 'SUNDIALS_TARGETS': '+'}, + 'pargs': {'flags': [], 'nargs': '1+'}}, + 'sundials_option': { 'kwargs': {'DEPENDS_ON': '+', 'OPTIONS': '+'}, + 'pargs': { 'flags': [ 'DEPENDS_ON_THROW_ERROR', + 'ADVANCED'], + 'nargs': '4+'}}, + 'sundials_trycompile_execute': { 'kwargs': { 'COMPILE_OUTPUT': 1, + 'RUN_OUTPUT': 1}, + 'pargs': {'flags': [], 'nargs': '4+'}}, + 'add_local_ci_target': {'pargs': {'nargs': 3}} + } + + # Override configurations per-command where available + override_spec = {} + + # Specify variable tags. + vartags = [] + + # Specify property tags. + proptags = [] + +# ----------------------------- +# Options affecting formatting. +# ----------------------------- +with section("format"): + + # Disable formatting entirely, making cmake-format a no-op + disable = False + + # How wide to allow formatted cmake files + line_width = 80 + + # How many spaces to tab for indent + tab_size = 2 + + # If true, lines are indented using tab characters (utf-8 0x09) instead of + # space characters (utf-8 0x20). In cases where the layout would + # require a fractional tab character, the behavior of the fractional + # indentation is governed by + use_tabchars = False + + # If is True, then the value of this variable indicates how + # fractional indentions are handled during whitespace replacement. If set to + # 'use-space', fractional indentation is left as spaces (utf-8 0x20). If set + # to `round-up` fractional indentation is replaced with a single tab character + # (utf-8 0x09) effectively shifting the column to the next tabstop + fractional_tab_policy = 'use-space' + + # If an argument group contains more than this many sub-groups (parg or kwarg + # groups) then force it to a vertical layout. + max_subgroups_hwrap = 2 + + # If a positional argument group contains more than this many arguments, then + # force it to a vertical layout. + max_pargs_hwrap = 6 + + # If a cmdline positional group consumes more than this many lines without + # nesting, then invalidate the layout (and nest) + max_rows_cmdline = 2 + + # If true, separate flow control names from their parentheses with a space + separate_ctrl_name_with_space = False + + # If true, separate function names from parentheses with a space + separate_fn_name_with_space = False + + # If a statement is wrapped to more than one line, than dangle the closing + # parenthesis on its own line. + dangle_parens = False + + # If the trailing parenthesis must be 'dangled' on its on line, then align it + # to this reference: `prefix`: the start of the statement, `prefix-indent`: + # the start of the statement, plus one indentation level, `child`: align to + # the column of the arguments + dangle_align = 'prefix' + + # If the statement spelling length (including space and parenthesis) is + # smaller than this amount, then force reject nested layouts. + min_prefix_chars = 4 + + # If the statement spelling length (including space and parenthesis) is larger + # than the tab width by more than this amount, then force reject un-nested + # layouts. + max_prefix_chars = 10 + + # If a candidate layout is wrapped horizontally but it exceeds this many + # lines, then reject the layout. + max_lines_hwrap = 2 + + # What style line endings to use in the output. + line_ending = 'unix' + + # Format command names consistently as 'lower' or 'upper' case + command_case = 'canonical' + + # Format keywords consistently as 'lower' or 'upper' case + keyword_case = 'unchanged' + + # A list of command names which should always be wrapped + always_wrap = [] + + # If true, the argument lists which are known to be sortable will be sorted + # lexicographicall + enable_sort = True + + # If true, the parsers may infer whether or not an argument list is sortable + # (without annotation). + autosort = False + + # By default, if cmake-format cannot successfully fit everything into the + # desired linewidth it will apply the last, most agressive attempt that it + # made. If this flag is True, however, cmake-format will print error, exit + # with non-zero status code, and write-out nothing + require_valid_layout = False + + # A dictionary mapping layout nodes to a list of wrap decisions. See the + # documentation for more information. + layout_passes = {} + +# ------------------------------------------------ +# Options affecting comment reflow and formatting. +# ------------------------------------------------ +with section("markup"): + + # What character to use for bulleted lists + bullet_char = '*' + + # What character to use as punctuation after numerals in an enumerated list + enum_char = '.' + + # If comment markup is enabled, don't reflow the first comment block in each + # listfile. Use this to preserve formatting of your copyright/license + # statements. + first_comment_is_literal = True + + # If comment markup is enabled, don't reflow any comment block which matches + # this (regex) pattern. Default is `None` (disabled). + literal_comment_pattern = None + + # Regular expression to match preformat fences in comments default= + # ``r'^\s*([`~]{3}[`~]*)(.*)$'`` + fence_pattern = '^\\s*([`~]{3}[`~]*)(.*)$' + + # Regular expression to match rulers in comments default= + # ``r'^\s*[^\w\s]{3}.*[^\w\s]{3}$'`` + ruler_pattern = '^\\s*[^\\w\\s]{3}.*[^\\w\\s]{3}$' + + # If a comment line matches starts with this pattern then it is explicitly a + # trailing comment for the preceeding argument. Default is '#<' + explicit_trailing_pattern = '#<' + + # If a comment line starts with at least this many consecutive hash + # characters, then don't lstrip() them off. This allows for lazy hash rulers + # where the first hash char is not separated by space + hashruler_min_length = 10 + + # If true, then insert a space between the first hash char and remaining hash + # chars in a hash ruler, and normalize its length to fill the column + canonicalize_hashrulers = True + + # enable comment markup parsing and reflow + enable_markup = True + +# ---------------------------- +# Options affecting the linter +# ---------------------------- +with section("lint"): + + # a list of lint codes to disable + disabled_codes = [] + + # regular expression pattern describing valid function names + function_pattern = '[0-9a-z_]+' + + # regular expression pattern describing valid macro names + macro_pattern = '[0-9A-Z_]+' + + # regular expression pattern describing valid names for variables with global + # (cache) scope + global_var_pattern = '[A-Z][0-9A-Z_]+' + + # regular expression pattern describing valid names for variables with global + # scope (but internal semantic) + internal_var_pattern = '_[A-Z][0-9A-Z_]+' + + # regular expression pattern describing valid names for variables with local + # scope + local_var_pattern = '[a-z][a-z0-9_]+' + + # regular expression pattern describing valid names for privatedirectory + # variables + private_var_pattern = '_[0-9a-z_]+' + + # regular expression pattern describing valid names for public directory + # variables + public_var_pattern = '[A-Z][0-9A-Z_]+' + + # regular expression pattern describing valid names for function/macro + # arguments and loop variables. + argument_var_pattern = '[a-z][a-z0-9_]+' + + # regular expression pattern describing valid names for keywords used in + # functions or macros + keyword_pattern = '[A-Z][0-9A-Z_]+' + + # In the heuristic for C0201, how many conditionals to match within a loop in + # before considering the loop a parser. + max_conditionals_custom_parser = 2 + + # Require at least this many newlines between statements + min_statement_spacing = 1 + + # Require no more than this many newlines between statements + max_statement_spacing = 2 + max_returns = 6 + max_branches = 12 + max_arguments = 5 + max_localvars = 15 + max_statements = 50 + +# ------------------------------- +# Options affecting file encoding +# ------------------------------- +with section("encode"): + + # If true, emit the unicode byte-order mark (BOM) at the start of the file + emit_byteorder_mark = False + + # Specify the encoding of the input file. Defaults to utf-8 + input_encoding = 'utf-8' + + # Specify the encoding of the output file. Defaults to utf-8. Note that cmake + # only claims to support utf-8 so be careful when using anything else + output_encoding = 'utf-8' + +# ------------------------------------- +# Miscellaneous configurations options. +# ------------------------------------- +with section("misc"): + + # A dictionary containing any per-command configuration overrides. Currently + # only `command_case` is supported. + per_command = {} + diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 51ab03ac92..870b004be9 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -4,3 +4,7 @@ fab1cecb7d91cff53b31730af5d00ff154c3b6ce cc6960349aa92e2bcad9168a6dacff99b21c329c # Apply formatting to Fortran files 23581e8454955283139e551a7bcd1b85d8b7c77b +# Apply formatting to Python files +b578eabccd77b7642b04ddda9d8530f05890d1b4 +# Apply formatting to CMake files +c6b9a02f24a27081c471d63dfc524684a9f5a9e3 diff --git a/.github/actions/apply-style/checkout.sh b/.github/actions/apply-style/checkout.sh new file mode 100755 index 0000000000..34a6f15b43 --- /dev/null +++ b/.github/actions/apply-style/checkout.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +### +# Attempt to find the branch of the PR from the detached head state +## + +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" +echo "Attempting to find branch that matches commit..." + +# Get the current commit SHA +current_commit_sha=$(git rev-parse HEAD) +# List all branches containing the current commit SHA +branches=$(git branch -r --contains $current_commit_sha) + +echo "all branches:" +echo "$(git branch -a)" +echo "branches with SHA $current_commit_sha:" +echo "$branches" + +# Loop over the string split by whitespace +branch="" +num_branches_found=0 +for _possible_branch in $branches; do + # Skip items that start with "pull/" + if [[ $_possible_branch == pull/* ]]; then + continue + fi + if [[ $_possible_branch == origin/* ]]; then + _possible_branch=$(echo "$_possible_branch" | sed 's/origin\///') + fi + echo "Possible Branch: $_possible_branch" + branch=$_possible_branch + num_branches_found=$((num_branches_found+1)) +done + +if [ "$num_branches_found" -ne 1 ]; then + echo "Error: Unable to find a single branch that matched git sha $current_commit_sha" + exit 1 +fi + +echo "Found branch: $branch" +echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + +git checkout $branch +git submodule update --init --recursive diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.yml index f56126095e..47c0fab83f 100644 --- a/.github/actions/test-driver/action.yml +++ b/.github/actions/test-driver/action.yml @@ -8,6 +8,9 @@ inputs: indexsize: description: SUNDIALS_INDEX_SIZE required: true + tpls: + description: "enable/disable TPLs" + required: true runs: using: composite @@ -26,5 +29,10 @@ runs: run: | git config --global --add safe.directory $GITHUB_WORKSPACE cd test - ./test_driver.sh --testtype CUSTOM --env env/docker.sh --tpls --sunrealtype ${{ inputs.precision }} --indexsize ${{ inputs.indexsize }} + ./test_driver.sh \ + --testtype CUSTOM \ + --env env/docker.sh \ + --tpls ${{ inputs.tpls }} \ + --sunrealtype ${{ inputs.precision }} \ + --indexsize ${{ inputs.indexsize }} shell: bash diff --git a/.github/dependabot.yml b/.github/dependabot.yml index df4d15b35c..3bc7efbf9d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,3 +6,4 @@ updates: schedule: # Check for updates to GitHub Actions every week interval: "weekly" + target-branch: "develop" diff --git a/.github/workflows/build-ci-containers-e4s.yml b/.github/workflows/build-ci-containers-e4s.yml index fe76443ae2..ee7c931f8d 100644 --- a/.github/workflows/build-ci-containers-e4s.yml +++ b/.github/workflows/build-ci-containers-e4s.yml @@ -26,7 +26,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Docker images - uses: docker/build-push-action@v6.0.2 + uses: docker/build-push-action@v6.10.0 with: context: "./docker/sundials-ci/e4s-base" build-args: e4s_version=22.05 @@ -56,7 +56,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Docker images - uses: docker/build-push-action@v6.0.2 + uses: docker/build-push-action@v6.10.0 with: context: "./docker/sundials-ci/e4s-quarterly" build-args: spack_yaml=./int${{ matrix.indexsize }}-${{ matrix.precision }}/spack.yaml diff --git a/.github/workflows/build-ci-containers-nightly.yml b/.github/workflows/build-ci-containers-nightly.yml index 7f9fdd9e3e..176b3a3d88 100644 --- a/.github/workflows/build-ci-containers-nightly.yml +++ b/.github/workflows/build-ci-containers-nightly.yml @@ -32,7 +32,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Docker images - uses: docker/build-push-action@v6.0.2 + uses: docker/build-push-action@v6.10.0 with: context: "./docker/sundials-ci/spack-nightly" build-args: spack_yaml=./int${{ matrix.indexsize }}-${{ matrix.precision }}/spack.yaml diff --git a/.github/workflows/check-clang-format.yml b/.github/workflows/check-clang-format.yml deleted file mode 100644 index 2a87a2a35d..0000000000 --- a/.github/workflows/check-clang-format.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Checks - clang-format - -on: - pull_request: - workflow_dispatch: - -jobs: - clang_format_check: - runs-on: ubuntu-latest - container: - image: ghcr.io/llnl/sundials_spack_cache:llvm-17.0.4-h4lflucc3v2vage45opbo2didtcuigsn.spack - steps: - - name: Install git - run: | - apt update - apt install -y git python3-pip - - - name: Install fprettify - run: pip install fprettify - - - name: Check out repository code - uses: actions/checkout@v4 - with: - submodules: true - - - name: Add safe directory - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - - - name: Print clang-format version - run: clang-format --version - - - name: Run checker on code - run: | - ./scripts/format.sh benchmarks examples include src test - - - name: Run git diff to see if anything changed - run: /usr/bin/git diff --name-only --exit-code - - - name: Run git diff if we failed - if: failure() - run: /usr/bin/git diff > clang_format.patch - - - name: Archive diff as a patch if we failed - uses: actions/upload-artifact@v3 - if: failure() - with: - name: clang_format.patch - path: | - ${{ github.workspace }}/clang_format.patch diff --git a/.github/workflows/check-format.yml b/.github/workflows/check-format.yml new file mode 100644 index 0000000000..f96e3cdaf7 --- /dev/null +++ b/.github/workflows/check-format.yml @@ -0,0 +1,120 @@ +name: Checks - formatting + +on: + pull_request: + workflow_dispatch: + issue_comment: + types: [created] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + format_check: + if: ${{ github.event_name != 'issue_comment' || (github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '/autofix')) }} + runs-on: ubuntu-latest + container: + image: ghcr.io/llnl/sundials_spack_cache:llvm-17.0.4-h4lflucc3v2vage45opbo2didtcuigsn.spack + steps: + - name: Install git + run: | + apt update + apt install -y git python3-pip + + - name: Install black + run: pip install black + + - name: Print black version + run: black --version + + - name: Install cmake-format + run: pip install cmakelang + + - name: Print cmake-format version + run: cmake-format --version + + - name: Install fprettify + run: pip install fprettify + + - name: Print fprettify version + run: fprettify --version + + - name: Print clang-format version + run: clang-format --version + + - name: Check out repository code + if: github.event_name != 'issue_comment' + uses: actions/checkout@v4 + with: + submodules: true + + - name: Check out repository code + if: github.event_name == 'issue_comment' + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.issue.number }}/head + fetch-depth: 0 + + - name: Add safe directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Run checker on code + run: | + ./scripts/format.sh CMakeLists.txt benchmarks cmake examples include src test + + - name: Run git diff to see if anything changed + run: /usr/bin/git diff --name-only --exit-code + + - name: Run git diff if we failed + if: failure() + run: /usr/bin/git diff > format.patch + + - name: Archive diff as a patch if we failed + uses: actions/upload-artifact@v4 + if: failure() + with: + name: format.patch + path: | + ${{ github.workspace }}/format.patch + + apply_format: + if: ${{ always() && contains(join(needs.*.result, ','), 'failure') && (github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '/autofix')) }} + needs: format_check + runs-on: ubuntu-latest + steps: + # Checkout the GitHub created reference for the PR. + # The only way to do this is by using the "issue" number + # but that is really the PR number in this context. + # This is due to using an `issue_comment` event which + # is due to the GitHub Actions API that does not have + # a `pull_request_comment` event or something similar. + # This leaves us in a detached head state which is corrected + # in `apply-style/checkout.sh` + - name: Checkout repository code + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.issue.number }}/head + fetch-depth: 0 + + - name: Checkout the right git ref + run: ./.github/actions/apply-style/checkout.sh + + - name: Download the git diff from format_check + uses: actions/download-artifact@v4 + with: + name: format.patch + + - name: Apply patch + run: | + git apply format.patch + rm format.patch + + - name: Commit fixes + run: | + git config user.name "format-robot" + git config user.email "no-reply@llnl.gov" + if [ -n "$(git status --porcelain)" ]; then + git commit -am 'apply format updates' + git push + fi diff --git a/.github/workflows/check-spelling.yml b/.github/workflows/check-spelling.yml new file mode 100644 index 0000000000..70da910142 --- /dev/null +++ b/.github/workflows/check-spelling.yml @@ -0,0 +1,100 @@ +name: Checks - spelling + +on: + pull_request: + workflow_dispatch: + issue_comment: + types: [created] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + spelling_check: + if: ${{ github.event_name != 'issue_comment' || (github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '/autofix')) }} + runs-on: ubuntu-latest + steps: + - name: Install python3-pip + run: | + sudo apt update + sudo apt install -y --no-install-recommends python3-pip + + - name: Install codespell + run: pip install codespell + + - name: Print codespell version + run: codespell --version + + - name: Check out repository code + if: github.event_name != 'issue_comment' + uses: actions/checkout@v4 + with: + submodules: true + + - name: Check out repository code + if: github.event_name == 'issue_comment' + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.issue.number }}/head + fetch-depth: 0 + + - name: Run codespell + run: | + ./scripts/spelling.sh + + - name: Run git diff to see if anything changed + run: /usr/bin/git diff --name-only --exit-code + + - name: Run git diff if we failed + if: failure() + run: /usr/bin/git diff > spelling.patch + + - name: Archive diff as a patch if we failed + uses: actions/upload-artifact@v4 + if: failure() + with: + name: spelling.patch + path: | + ${{ github.workspace }}/spelling.patch + + apply_spelling: + if: ${{ always() && contains(join(needs.*.result, ','), 'failure') && (github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '/autofix')) }} + needs: spelling_check + runs-on: ubuntu-latest + steps: + # Checkout the GitHub created reference for the PR. + # The only way to do this is by using the "issue" number + # but that is really the PR number in this context. + # This is due to using an `issue_comment` event which + # is due to the GitHub Actions API that does not have + # a `pull_request_comment` event or something similar. + # This leaves us in a detached head state which is corrected + # in `apply-style/checkout.sh` + - name: Checkout repository code + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.issue.number }}/head + fetch-depth: 0 + + - name: Checkout the right git ref + run: ./.github/actions/apply-style/checkout.sh + + - name: Download the git diff from spelling_check + uses: actions/download-artifact@v4 + with: + name: spelling.patch + + - name: Apply patch + run: | + git apply spelling.patch + rm spelling.patch + + - name: Commit fixes + run: | + git config user.name "format-robot" + git config user.email "no-reply@llnl.gov" + if [ -n "$(git status --porcelain)" ]; then + git commit -am 'apply spelling updates' + git push + fi diff --git a/.github/workflows/check-swig.yml b/.github/workflows/check-swig.yml index 2a4193a239..2f9fad653d 100644 --- a/.github/workflows/check-swig.yml +++ b/.github/workflows/check-swig.yml @@ -3,11 +3,17 @@ name: Checks - swig on: pull_request: workflow_dispatch: + issue_comment: + types: [created] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true jobs: - swig: + swig_check: + if: ${{ github.event_name != 'issue_comment' || (github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '/autofix')) }} runs-on: ubuntu-latest - steps: - name: Install swig run: | @@ -15,15 +21,23 @@ jobs: cd swig ./autogen.sh ./configure --prefix=/usr/ - make + make sudo make install swig -version - name: Check out repository code + if: github.event_name != 'issue_comment' uses: actions/checkout@v4 with: submodules: true + - name: Check out repository code + if: github.event_name == 'issue_comment' + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.issue.number }}/head + fetch-depth: 0 + - name: Add safe directory run: git config --global --add safe.directory "$GITHUB_WORKSPACE" @@ -41,9 +55,50 @@ jobs: run: /usr/bin/git diff > swig.patch - name: Archive diff as a patch if we failed - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: swig.patch path: | ${{ github.workspace }}/swig.patch + + apply_swig: + if: ${{ always() && contains(join(needs.*.result, ','), 'failure') && (github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '/autofix')) }} + needs: swig_check + runs-on: ubuntu-latest + steps: + # Checkout the GitHub created reference for the PR. + # The only way to do this is by using the "issue" number + # but that is really the PR number in this context. + # This is due to using an `issue_comment` event which + # is due to the GitHub Actions API that does not have + # a `pull_request_comment` event or something similar. + # This leaves us in a detached head state which is corrected + # in `apply-style/checkout.sh` + - name: Checkout repository code + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.issue.number }}/head + fetch-depth: 0 + + - name: Checkout the right git ref + run: ./.github/actions/apply-style/checkout.sh + + - name: Download the git diff from swig_check + uses: actions/download-artifact@v4 + with: + name: swig.patch + + - name: Apply patch + run: | + git apply swig.patch + rm swig.patch + + - name: Commit fixes + run: | + git config user.name "format-robot" + git config user.email "no-reply@llnl.gov" + if [ -n "$(git status --porcelain)" ]; then + git commit -am 'apply swig updates' + git push + fi diff --git a/.github/workflows/spack-develop.yml b/.github/workflows/spack-develop.yml index 41f312164f..adf15d2de3 100644 --- a/.github/workflows/spack-develop.yml +++ b/.github/workflows/spack-develop.yml @@ -33,7 +33,7 @@ jobs: indexsize: ${{ matrix.indexsize }} precision: ${{ matrix.precision }} - name: Archive build files from failed build - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: build_files @@ -41,7 +41,7 @@ jobs: ${{ github.workspace }}/test/build_* !${{ github.workspace }}/test/build_*/Testing/output - name: Archive output files from failed build - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: output_files diff --git a/.github/workflows/test-address-sanitizer.yml b/.github/workflows/test-address-sanitizer.yml new file mode 100644 index 0000000000..bb2aa80324 --- /dev/null +++ b/.github/workflows/test-address-sanitizer.yml @@ -0,0 +1,59 @@ +# +name: Build and Test - Ubuntu/gcc (Address Sanitizer) + +on: + push: + branches: + - main + - develop + pull_request: + merge_group: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + sanitizer_build_and_test: + runs-on: ubuntu-latest + container: + image: ghcr.io/llnl/sundials-ci-int${{ matrix.indexsize }}-${{ matrix.precision }}:latest + options: --user root + strategy: + max-parallel: 2 + matrix: + indexsize: [32, 64] + precision: ['double'] + buildtype: ['Debug'] + # Address sanitizer is enabled in test_driver.sh with TPLs OFF + tpls: ['OFF'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + - name: Run test_driver.sh + uses: ./.github/actions/test-driver + with: + indexsize: ${{ matrix.indexsize }} + precision: ${{ matrix.precision }} + tpls: ${{ matrix.tpls }} + env: + CMAKE_BUILD_TYPE: ${{ matrix.buildtype }} + - name: Archive build files from failed build + uses: actions/upload-artifact@v4 + if: failure() + with: + name: build_files + path: | + ${{ github.workspace }}/test/build_* + !${{ github.workspace }}/test/build_*/Testing/output + - name: Archive output files from failed build + uses: actions/upload-artifact@v4 + if: failure() + with: + name: output_files + path: | + ${{ github.workspace }}/test/build_*/Testing/ diff --git a/.github/workflows/ubuntu-clang-latest.yml b/.github/workflows/ubuntu-clang-latest.yml index 2010b54dbf..57b3dfc85e 100644 --- a/.github/workflows/ubuntu-clang-latest.yml +++ b/.github/workflows/ubuntu-clang-latest.yml @@ -43,12 +43,25 @@ jobs: -D CMAKE_CXX_COMPILER=$(which clang++) \ -D SUNDIALS_LOGGING_LEVEL=${{matrix.logging_level}} \ -D ENABLE_ALL_WARNINGS=ON \ - -D ENABLE_WARNINGS_AS_ERRORS=ON + -D ENABLE_WARNINGS_AS_ERRORS=ON \ + -D SUNDIALS_TEST_DEVTESTS=ON \ + -D SUNDIALS_TEST_UNITTESTS=ON \ + -D SUNDIALS_TEST_ENABLE_GTEST=OFF \ + -D SUNDIALS_TEST_FLOAT_PRECISION=4 \ + -D SUNDIALS_TEST_INTEGER_PRECISION=10 - name: Build # Build your program with the given configuration run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} + - name: Test + # Run logging tests + run: | + ctest \ + --test-dir ${{github.workspace}}/build \ + --label-regex logging \ + --verbose + build_cycle_profiling: runs-on: ubuntu-latest diff --git a/.github/workflows/ubuntu-latest-oneapi.yml b/.github/workflows/ubuntu-latest-oneapi.yml new file mode 100644 index 0000000000..bebd04efad --- /dev/null +++ b/.github/workflows/ubuntu-latest-oneapi.yml @@ -0,0 +1,45 @@ +name: Build - Ubuntu/dpcpp + +on: + pull_request: + merge_group: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +env: + # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) + BUILD_TYPE: Release + +jobs: + build: + strategy: + matrix: + ONEAPI_VERSION: [ + '2024.2.1-0-devel-ubuntu22.04', + 'latest' # 2025.0.0-0-devel-ubuntu24.04 at the time of commit + ] + + runs-on: ubuntu-latest + container: intel/oneapi-basekit:${{ matrix.ONEAPI_VERSION }} + + steps: + - uses: actions/checkout@v4 + + - name: Configure CMake + run: | + cmake \ + -B ${{github.workspace}}/build \ + -D CMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \ + -D CMAKE_C_COMPILER=$(which icx) \ + -D CMAKE_CXX_COMPILER=$(which icpx) \ + -D CMAKE_CXX_FLAGS="-fsycl" \ + -D SUNDIALS_BUILD_WITH_PROFILING=ON \ + -D ENABLE_ALL_WARNINGS=ON \ + -D ENABLE_WARNINGS_AS_ERRORS=ON \ + -D ENABLE_SYCL=ON + + - name: Build + run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} diff --git a/.github/workflows/ubuntu-latest.yml b/.github/workflows/ubuntu-latest.yml index 0bc9ab7510..c63b2125be 100644 --- a/.github/workflows/ubuntu-latest.yml +++ b/.github/workflows/ubuntu-latest.yml @@ -27,6 +27,7 @@ jobs: # Disable extended tests until compiler warnings are addressed precision: ['single', 'double'] buildtype: ['Debug', 'Release', 'RelWithDebInfo'] + tpls: ['ON'] exclude: - buildtype: Debug precision: single @@ -49,10 +50,11 @@ jobs: with: indexsize: ${{ matrix.indexsize }} precision: ${{ matrix.precision }} + tpls: ${{ matrix.tpls }} env: CMAKE_BUILD_TYPE: ${{ matrix.buildtype }} - name: Archive build files from failed build - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: build_files @@ -60,7 +62,7 @@ jobs: ${{ github.workspace }}/test/build_* !${{ github.workspace }}/test/build_*/Testing/output - name: Archive output files from failed build - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: output_files diff --git a/.gitignore b/.gitignore index 3e9185713d..fcaaffcbd6 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,8 @@ compile_commands.json *.orig *.nvvp *.ptp-sync* +*.project +*.cproject *~ \#* @@ -75,3 +77,6 @@ uberenv_libs # swig /swig/swig + +# tools +/tools/suntools/__pycache__ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c02b239373..b748728bc2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,23 @@ -# --------------------------------------------------------------- +############################################################################### +# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# SPDX-License-Identifier: (MIT) +############################################################################### +# Modifications to this file have been made under the folllowing copyright. +# ----------------------------------------------------------------------------- # SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security +# Copyright (c) 2002-2024, Lawrence Livermore National Security # and Southern Methodist University. # All rights reserved. # @@ -8,129 +25,97 @@ # # SPDX-License-Identifier: BSD-3-Clause # SUNDIALS Copyright End -# --------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# DESCRIPTION: ############################################################################### # General GitLab pipelines configurations for supercomputers and Linux clusters # at Lawrence Livermore National Laboratory (LLNL). -############################################################################### - -# We define the following GitLab pipeline variables (all of which can be -# set from the GitLab CI "run pipeline" UI): -# -# GIT_SUBMODULE_STRATEGY: -# Tells Gitlab to recursively update the submodules when cloning. -# -# ALLOC_NAME: -# Allocation unique name. -# -# BUILD_ROOT: -# The path to the shared resources between all jobs. The BUILD_ROOT is unique to -# the pipeline, preventing any form of concurrency with other pipelines. This -# also means that the BUILD_ROOT directory will never be cleaned. -# -# DEFAULT_TIME: -# Default time to let the jobs run will be 30 minutes. -# -# BUILD_JOBS: -# Number of threads to use for builds. -# -# NCPU: -# Number of CPUs to alloc. +# This entire pipeline is LLNL-specific # -# VERBOSE_BUILD: -# Sets CMAKE_VERBOSE_MAKEFILE to TRUE when "ON". +# Important note: This file is a template provided by llnl/radiuss-shared-ci. +# Remains to set variable values, change the reference to the radiuss-shared-ci +# repo, opt-in and out optional features. The project can then extend it with +# additional stages. # -# VERBOSE_TEST: -# Passes --verbose to CTest when "ON". +# In addition, each project should copy over and complete: +# - .gitlab/custom-jobs-and-variables.yml +# - .gitlab/subscribed-pipelines.yml # -# ON_LASSEN: -# Should the Lassen pipeline run? Set to "ON" or "OFF" to enable/disable. -# -# ON_QUARTZ: -# Should the Quartz pipeline run? Set to "ON" or "OFF" to enable/disable. -# -# SPACK_PREFIX: prefix used for shared spack installation. -# Usually this a spack version number that matches the version set in the uberenv_config.json file. -# Spack installs go in /usr/workspace/sundials/spack_installs/${SPACK_PREFIX}/$(hostname). -# -# SHARED_SPACK: -# If "ON", then a shared spack install that has been pre-configured is utilized. -# If "OFF", then a new spack instance is created for every build (meaning all TPLs have to be installed). -# If "UPSTREAM" (the default), then the shared spack is used as an upstream for a build specific spack. -# -# BENCHMARK: -# If "ON", then the SUNDIALS benchmark problems are run and generate profiles. -# -# BENCHMARK_NNODES: -# Number of nodes to use for benchmarks. Default is 4. -# -# BENCHMARK_QUEUE: -# What queue to submit the benchmarks too. Default is pbatch (for Livermore). +# The jobs should be specified in a file local to the project, +# - .gitlab/jobs/${CI_MACHINE}.yml +# or generated (see LLNL/Umpire for an example). +############################################################################### +# We define the following GitLab pipeline variables: variables: +##### LC GITLAB CONFIGURATION +# Use an LLNL service user to run CI. This prevents from running pipelines as +# an actual user. + LLNL_SERVICE_USER: "" +# Use the service user workspace. Solves permission issues, stores everything +# at the same location whoever triggers a pipeline. + CUSTOM_CI_BUILDS_DIR: "/usr/workspace/sundials/ci/.builds/" +# Tells Gitlab to recursively update the submodules when cloning the project. GIT_SUBMODULE_STRATEGY: recursive - ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} - BUILD_ROOT: ${CI_PROJECT_DIR} - DEFAULT_TIME: 30 - BUILD_JOBS: 32 - NCPUS: 12 - VERBOSE_BUILD: "OFF" - VERBOSE_TEST: "OFF" - ON_LASSEN: "ON" - ON_QUARTZ: "ON" - ON_CORONA: "ON" - SPACK_PREFIX: "v0.19.1" - SHARED_SPACK: "UPSTREAM" - BENCHMARK: "OFF" - BENCHMARK_NNODES: 4 - BENCHMARK_QUEUE: "pbatch" - -# Normally, stages are blocking in Gitlab. However, using the keyword "needs" we -# can express dependencies between job that break the ordering of stages, in -# favor of a DAG. -# In practice q_*, l_* and c_* stages are independently run and start immediately. -stages: - - q_build_and_test - - l_build_and_test - - l_build_and_bench - - c_build_and_test +##### PROJECT VARIABLES +# We build the projects in the CI clone directory. +# Used in script/gitlab/build_and_test.sh script. +# TODO: add a clean-up mechanism. + BUILD_ROOT: ${CI_PROJECT_DIR} + SPACK_REF: "594a376c521cc746978571b1181a47bbcff30a21" # v0.22.2 -# These are also templates (.name) that define project specific build commands. -# If an allocation exist with the name defined in this pipeline, the job will -# use it (slurm specific). -.build_toss_3_x86_64_ib_script: - script: - - echo ${ALLOC_NAME} - - srun -p pdebug -N 1 -n ${NCPUS} --interactive -t ${DEFAULT_TIME} - --job-name=${ALLOC_NAME} .gitlab/build_and_test.sh +##### SHARED_CI CONFIGURATION +# Required information about GitHub repository + GITHUB_PROJECT_NAME: "sundials" + GITHUB_PROJECT_ORG: "LLNL" +# Set the build-and-test command. +# Nested variables are allowed and useful to customize the job command. We +# prevent variable expansion so that you can define them at job level. + JOB_CMD: + value: ".gitlab/build_and_test.sh" + expand: false +# Override the pattern describing branches that will skip the "draft PR filter +# test". Add protected branches here. See default value in +# preliminary-ignore-draft-pr.yml. +# ALWAYS_RUN_PATTERN: "" -# Corona -.build_toss_4_x86_64_ib_corona_script: - script: - - echo ${ALLOC_NAME} - - flux alloc -N 1 -t ${DEFAULT_TIME} .gitlab/build_and_test.sh +# We organize the build-and-test stage with sub-pipelines. Each sub-pipeline +# corresponds to a test batch on a given machine. -# CORAL systems use spectrum LSF instead of SLURM -.build_blueos_3_ppc64le_ib_script: - script: - - echo ${ALLOC_NAME} - - bsub -q pdebug -J ${ALLOC_NAME} -nnodes 1 -W ${DEFAULT_TIME} -Is .gitlab/build_and_test.sh +# High level stages +stages: + - prerequisites + - build-and-test -# Benchmark job for CORAL systems -.build_blueos_3_ppc64le_ib_bench: - script: - - echo ${ALLOC_NAME} - - bsub -q ${BENCHMARK_QUEUE} -J ${ALLOC_NAME} -nnodes ${BENCHMARK_NNODES} -W ${DEFAULT_TIME} -Is .gitlab/build_and_bench.sh +# Template for jobs triggering a build-and-test sub-pipeline: +.build-and-test: + stage: build-and-test + trigger: + include: + - local: '.gitlab/custom-jobs-and-variables.yml' + - project: 'sundials/radiuss-shared-ci' # https://lc.llnl.gov/gitlab/sundials/radiuss-shared-ci + ref: 'ae1f3786591beed83abc6a0de2229f6e9532e2d4' + file: 'pipelines/${CI_MACHINE}.yml' + # Add your jobs + # you can use a local file + - local: '.gitlab/jobs/${CI_MACHINE}.yml' + # or a file generated in the previous steps + # - artifact: '${CI_MACHINE}-jobs.yml' + # job: 'generate-job-file' + # (See Umpire CI setup for an example). + strategy: depend + forward: + pipeline_variables: true -# This is where jobs are included. include: + # Sets ID tokens for every job using `default:` - project: 'lc-templates/id_tokens' file: 'id_tokens.yml' - - local: .gitlab/quartz-templates.yml - - local: .gitlab/quartz-jobs.yml - - local: .gitlab/lassen-templates.yml - - local: .gitlab/lassen-jobs.yml - - local: .gitlab/corona-templates.yml - - local: .gitlab/corona-jobs.yml + # # [Optional] checks preliminary to running the actual CI test + # - project: 'radiuss/radiuss-shared-ci' + # ref: 'v2024.04.0' + # file: 'utilities/preliminary-ignore-draft-pr.yml' + # pipelines subscribed by the project + - local: '.gitlab/subscribed-pipelines.yml' diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index cf7ab193fa..04abb537ea 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -20,9 +20,6 @@ job_unique_id=${CI_JOB_ID:-""} sys_type=${SYS_TYPE:-""} py_env_path=${PYTHON_ENVIRONMENT_PATH:-""} -spack_prefix=${SHARED_SPACK_PREFIX:-"v0.19.1"} -shared_spack=${SHARED_SPACK:-"UPSTREAM"} - # Dependencies date @@ -38,7 +35,6 @@ echo "spec = ${spec}" echo "job_unique_id = ${job_unique_id}" echo "sys_type = ${sys_type}" echo "py_env_path = ${py_env_path}" -echo "shared_spack = ${shared_spack}" # remove tailing number from hostname hostname=${hostname%%[0-9]*} @@ -46,19 +42,6 @@ hostname=${hostname%%[0-9]*} # number of parallel build jobs BUILD_JOBS=${BUILD_JOBS:-"1"} -# load newer python to try the clingo concretizer -# machine specific loads -if [[ "${hostname}" == "lassen" ]]; then - echo "module load python/3.8.2" - module load python/3.8.2 -elif [[ "${hostname}" == "corona" ]]; then - echo "module load python/3.9.12" - module load python/3.9.12 -else - echo "module load python" - module load python -fi - if [[ "${option}" != "--build-only" && "${option}" != "--test-only" ]] then echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" @@ -97,35 +80,29 @@ then mkdir -p ${spack_user_cache} fi - if [[ -d /usr/workspace/sundials ]] - then - upstream="/usr/workspace/sundials/spack_installs/${spack_prefix}/${hostname}" - mkdir -p "${upstream}" - upstream_opt="--upstream=${upstream}" - fi + mirror_opt="" + buildcache="/usr/workspace/sundials/ci/spack_stuff/build_caches/${SPACK_REF}" - if [[ "${shared_spack}" == "UPSTREAM" ]] + if [[ ! -d "${buildcache}" ]] then - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" "${upstream_opt}" - elif [[ "${shared_spack}" == "ON" ]] - then - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" --prefix="${upstream}" - else - python3 .gitlab/uberenv/uberenv.py --spec="${spec}" "${prefix_opt}" + mkdir "${buildcache}" fi - # Ensure correct CUDA module is loaded, only works for module naming - # convention on Lassen. Only needed for CUDA 11 (unclear why). - if [[ -n "${CUDA_SPEC}" ]]; then - cuda_version="${CUDA_SPEC##*@}" - echo "module load cuda/${cuda_version}" - module load cuda/"${cuda_version}" - fi + mirror_opt=("--mirror=${buildcache}" "--mirror-autopush") - module load cmake/3.23 + key_path=/usr/workspace/sundials/ci/spack_stuff/gpg_backup + python3 .gitlab/uberenv/uberenv.py \ + --trust-key ${key_path}/pubring.gpg --trust-key ${key_path}/secring.gpg \ + --spec="${spec}" "${mirror_opt[@]}" "${prefix_opt}" \ + --spack-commit="${SPACK_REF}" fi + date +# Reload the spack environment created by uberenv +. ${prefix}/spack/share/spack/setup-env.sh +spack load + # Host config file if [[ -z ${hostconfig} ]] then @@ -173,6 +150,9 @@ then echo "~ Host-config: ${hostconfig_path}" echo "~ Build Dir: ${build_dir}" echo "~ Project Dir: ${project_dir}" + echo "~ MPIEXEC_EXECUTABLE: ${MPIEXEC_EXECUTABLE}" + echo "~ MPIEXEC_PREFLAGS: ${MPIEXEC_PREFLAGS}" + echo "~ MPIEXEC_POSTFLAGS: ${MPIEXEC_POSTFLAGS}" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" @@ -195,6 +175,9 @@ then $cmake_exe \ -C "${hostconfig_path}" \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ + -DMPIEXEC_EXECUTABLE=$(which $MPIEXEC_EXECUTABLE) \ + -DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS} \ + -DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS} \ -DSUNDIALS_CALIPER_OUTPUT_DIR="${CALIPER_DIR}/Release/${hostname}/${sundials_version}" \ "${project_dir}" @@ -202,6 +185,9 @@ then $cmake_exe \ -C "${hostconfig_path}" \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ + -DMPIEXEC_EXECUTABLE=$(which $MPIEXEC_EXECUTABLE) \ + -DMPIEXEC_PREFLAGS=${MPIEXEC_PREFLAGS} \ + -DMPIEXEC_POSTFLAGS=${MPIEXEC_POSTFLAGS} \ "${project_dir}" fi diff --git a/.gitlab/corona-jobs.yml b/.gitlab/corona-jobs.yml deleted file mode 100644 index f694611c39..0000000000 --- a/.gitlab/corona-jobs.yml +++ /dev/null @@ -1,48 +0,0 @@ -# ------------------------------------------------------------------------------ -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# ------------------------------------------------------------------------------ - -# ------------------------------------------------------------------------------ -# HIP -# ------------------------------------------------------------------------------ - -# Builds with HIP -corona_rocmcc_550: - parallel: - matrix: - - COMPILER_SPEC: rocmcc@5.5.0 - AMDGPU_TARGET: [gfx906] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi" - extends: .corona_build_and_test - -# ------------------------------------------------------------------------------ -# HIP + TPLs -# ------------------------------------------------------------------------------ -corona_rocmcc_550_tpls: - parallel: - matrix: - - COMPILER_SPEC: rocmcc@5.5.0 - AMDGPU_TARGET: [gfx906] - ROCM_VERSION: 5.5.0 - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja+kokkos+kokkos-kernels~ginkgo ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm~profiling amdgpu_target=${AMDGPU_TARGET} ^hipblas@${ROCM_VERSION} ^hipsparse@${ROCM_VERSION} ^hip@${ROCM_VERSION} ^hsa-rocr-dev@${ROCM_VERSION} ^llvm-amdgpu@${ROCM_VERSION}" - extends: .corona_build_and_test - -corona_rocmcc_523_tpls: - parallel: - matrix: - - COMPILER_SPEC: rocmcc@5.2.3 - AMDGPU_TARGET: [gfx906] - ROCM_VERSION: 5.2.3 - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 amdgpu_target=${AMDGPU_TARGET} scheduler=flux +rocm+mpi+magma+raja+kokkos+kokkos-kernels+ginkgo ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm~profiling amdgpu_target=${AMDGPU_TARGET} ^ginkgo+rocm amdgpu_target=${AMDGPU_TARGET} ^hipblas@${ROCM_VERSION} ^hipsparse@${ROCM_VERSION} ^rocrand@${ROCM_VERSION} ^rocthrust@${ROCM_VERSION} ^hip@${ROCM_VERSION} ^hsa-rocr-dev@${ROCM_VERSION} ^llvm-amdgpu@${ROCM_VERSION} ^rocprim@${ROCM_VERSION}" - extends: .corona_build_and_test diff --git a/.gitlab/corona-templates.yml b/.gitlab/corona-templates.yml deleted file mode 100644 index a901f4fa48..0000000000 --- a/.gitlab/corona-templates.yml +++ /dev/null @@ -1,35 +0,0 @@ -# ------------------------------------------------------------------------------ -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# ------------------------------------------------------------------------------ - -# ------------------------------------------------------------------------------ -# Tags and rules to run tests on Corona -# ------------------------------------------------------------------------------ - -# Generic Corona build job, extending build script for Toss 4 x86_64 Systems -.corona_build_and_test: - tags: - - shell - - corona - extends: [.build_toss_4_x86_64_ib_corona_script] - stage: c_build_and_test - needs: [] - artifacts: - paths: - - spack-*.txt - - build_*/* - when: always - rules: - # Don't run if... - - if: '$CI_COMMIT_BRANCH =~ /_cnone/ || $ON_CORONA == "OFF" || $BENCHMARK == "ON"' - when: never - # Default is to run if previous stage succeeded - - when: on_success diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml new file mode 100644 index 0000000000..1b830e2e3c --- /dev/null +++ b/.gitlab/custom-jobs-and-variables.yml @@ -0,0 +1,84 @@ +############################################################################### +# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# SPDX-License-Identifier: (MIT) +############################################################################### +# Modifications to this file have been made under the folllowing copyright. +# ----------------------------------------------------------------------------- +# SUNDIALS Copyright Start +# Copyright (c) 2002-2024, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ----------------------------------------------------------------------------- + +# We define the following GitLab pipeline variables: +variables: +# In some pipelines we create only one allocation shared among jobs in +# order to save time and resources. This allocation has to be uniquely +# named so that we are sure to retrieve it and avoid collisions. + ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} + +# Dane +# Arguments for top level allocation + DANE_SHARED_ALLOC: "--exclusive --reservation=ci --time=60 --nodes=1" +# Arguments for job level allocation + DANE_JOB_ALLOC: "--time=55 --nodes=1 --overlap" +# Add variables that should apply to all the jobs on a machine: +# DANE_MY_VAR: "..." + +# Tioga +# Arguments for top level allocation + TIOGA_SHARED_ALLOC: "--exclusive --queue pci --time-limit=60m --nodes=1" +# Arguments for job level allocation + TIOGA_JOB_ALLOC: "--time-limit=55m --nodes=1" +# Add variables that should apply to all the jobs on a machine: +# TIOGA_MY_VAR: "..." + +# Number of threads to use for builds. + BUILD_JOBS: 32 + +# Sets CMAKE_VERBOSE_MAKEFILE to TRUE when "ON". + VERBOSE_BUILD: "ON" + +# Passes --verbose to CTest when "ON". + VERBOSE_TEST: "ON" + +# Should the Dane pipeline run? Set to "ON" or "OFF" to enable/disable. + ON_DANE: "ON" + +# Should the Tioga pipeline run? Set to "ON" or "OFF" to enable/disable. + ON_TIOGA: "ON" + +# If "ON", then the SUNDIALS benchmark problems are run and generate profiles. + BENCHMARK: "OFF" + +# Number of nodes to use for benchmarks. Default is 4. + BENCHMARK_NNODES: 4 + +# What queue to submit the benchmarks too. Default is pbatch (for Livermore). + BENCHMARK_QUEUE: "pbatch" + +# Configuration shared by build and test jobs specific to this project. +# Not all configuration can be shared. Here projects can fine tune the +# CI behavior. +# See Umpire for an example (export junit test reports). +.custom_job: + variables: + SPACK_DISABLE_LOCAL_CONFIG: "true" + SPACK_USER_CACHE_PATH: /tmp/spack diff --git a/.gitlab/jobs/dane.yml b/.gitlab/jobs/dane.yml new file mode 100644 index 0000000000..bcb31c8bc8 --- /dev/null +++ b/.gitlab/jobs/dane.yml @@ -0,0 +1,66 @@ +############################################################################## +# Copyright (c) 2022-24, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################## +# ------------------------------------------------------------------------------ +# SUNDIALS Copyright Start +# Copyright (c) 2002-2024, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ------------------------------------------------------------------------------ + +########################################## +# Overridden shared radiuss-shared-ci jobs +########################################## + +# We duplicate the shared jobs description and add necessary changes for this +# project. We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} +# So that the comparison with the original job is easier. + +# No overridden jobs so far. + +######################## +# Extra jobs +######################## + +.sundials_job_on_dane: + variables: + MPIEXEC_EXECUTABLE: "srun" + extends: [.custom_job, .job_on_dane] + +dane_clang_tpls: + parallel: + matrix: + - COMPILER_SPEC: clang@14.0.6 + INDEX_SPEC: [~int64] + PRECISION_SPEC: [double] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +f2003+mpi+openmp+hypre+superlu-dist+lapack+klu+petsc+ginkgo+kokkos+kokkos-kernels~trilinos ^openblas" + extends: .sundials_job_on_dane + +dane_gcc_tpls: + parallel: + matrix: + - COMPILER_SPEC: gcc@10.3.1 + INDEX_SPEC: [~int64] + PRECISION_SPEC: [double] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +f2003+mpi+openmp+hypre+superlu-dist+lapack+klu+petsc+ginkgo+kokkos+kokkos-kernels~trilinos ^netlib-lapack" + extends: .sundials_job_on_dane + +dane_intel_tpls: + parallel: + matrix: + - COMPILER_SPEC: intel@2023.2.1 + INDEX_SPEC: [~int64] + PRECISION_SPEC: [double] + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} ~f2003+mpi+openmp+hypre+superlu-dist+lapack~klu+petsc+ginkgo+kokkos+kokkos-kernels~trilinos ^intel-oneapi-mkl" + extends: .sundials_job_on_dane diff --git a/.gitlab/jobs/tioga.yml b/.gitlab/jobs/tioga.yml new file mode 100644 index 0000000000..3a38476cca --- /dev/null +++ b/.gitlab/jobs/tioga.yml @@ -0,0 +1,92 @@ +############################################################################## +# Copyright (c) 2022-24, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. See the COPYRIGHT file for details. +# +# SPDX-License-Identifier: (MIT) +############################################################################## +# ------------------------------------------------------------------------------ +# SUNDIALS Copyright Start +# Copyright (c) 2002-2024, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ------------------------------------------------------------------------------ + +########################################## +# Overridden shared radiuss-shared-ci jobs +########################################## + +# We duplicate the shared jobs description and add necessary changes for this +# project. We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} +# So that the comparison with the original job is easier. + +# No overridden jobs so far. + +######################## +# Extra jobs +######################## + +.sundials_job_on_tioga: + variables: + MPIEXEC_EXECUTABLE: "flux" + MPIEXEC_PREFLAGS: "run" + extends: [.custom_job, .job_on_tioga] + +# --- Builds with HIP + +# Builds fine but non-mpi tests fail to execute due to a missing so: +# error while loading shared libraries: libpgmath.so: cannot open shared object file: No such file or directory +# This seems like it might be an LC environment issue, so disabling the job for now. +tioga_rocmcc_571_tpls: + parallel: + matrix: + - COMPILER_SPEC: rocmcc@5.7.1 + AMDGPU_TARGET: [gfx90a] + variables: + ON_TIOGA: "OFF" # disable until we can figure out libpgmath.so error + # have to use ginkgo@master because our spack version does not have ginkgo@1.8.0: yet (which seems to be needed) + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} +rocm+mpi+magma+ginkgo+kokkos ^magma+rocm amdgpu_target=${AMDGPU_TARGET} ^ginkgo@master+rocm amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm amdgpu_target=${AMDGPU_TARGET}" + before_script: + - module load rocmcc/5.7.1-magic + extends: [.sundials_job_on_tioga] + +tioga_rocmcc_620_tpls: + parallel: + matrix: + - COMPILER_SPEC: rocmcc@6.2.0 + AMDGPU_TARGET: [gfx90a] + variables: + # have to use ginkgo@master because our spack version does not have ginkgo@1.8.0: yet (which seems to be needed) + # similarly, we need a newer magma than available to compile with 'rocm@6:' so we turn it off + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double amdgpu_target=${AMDGPU_TARGET} +rocm+mpi~magma+ginkgo+kokkos ^ginkgo@master+rocm amdgpu_target=${AMDGPU_TARGET} ^kokkos+rocm amdgpu_target=${AMDGPU_TARGET}" + before_script: + - module load rocmcc/6.2.0-magic + extends: [.sundials_job_on_tioga] + +# --- Builds without HIP + +tioga_cce_1800_mpi_only: + parallel: + matrix: + - COMPILER_SPEC: cce@18.0.0 + variables: + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double +mpi" + extends: [.sundials_job_on_tioga] + + +# --- Benchmark jobs + +tioga_rocmcc_620_benchmark: + parallel: + matrix: + - COMPILER_SPEC: rocmcc@6.2.0 + AMDGPU_TARGET: [gfx90a] + CALIPER_DIR: /usr/workspace/sundials/ci/performance/califiles + variables: + ON_TIOGA: "OFF" # disable until we re-baseline on Tioga + SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double scheduler=flux caliper-dir=${CALIPER_DIR} ~int64 +benchmarks+profiling+caliper+adiak+mpi+openmp+rocm+raja amdgpu_target=${AMDGPU_TARGET} ^raja+rocm~openmp~examples~exercises amdgpu_target=${AMDGPU_TARGET} ^caliper+adiak+rocm amdgpu_target=${AMDGPU_TARGET}" + extends: [.sundials_job_on_tioga] diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml deleted file mode 100644 index 1ffca71d46..0000000000 --- a/.gitlab/lassen-jobs.yml +++ /dev/null @@ -1,45 +0,0 @@ -# ------------------------------------------------------------------------------ -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# ------------------------------------------------------------------------------ - -# Builds with CUDA, RAJA and other TPLs -lassen_cuda_no_tpls: - parallel: - matrix: - - COMPILER_SPEC: [xl@16.1.1.14] - CUDA_SPEC: [cuda@10.1.243] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda~raja~magma~superlu-dist~petsc~hypre~ginkgo cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" - extends: .lassen_build_and_test - -lassen_gcc_cuda_tpls: - parallel: - matrix: - - COMPILER_SPEC: gcc@8.3.1 - CUDA_SPEC: [cuda@11.8.0] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 precision=double ~int64 +mpi+openmp+cuda+raja+magma+superlu-dist+petsc+hypre+ginkgo cuda_arch=70 ^ginkgo+cuda cuda_arch=70 ^hypre~cuda ^petsc~cuda ^superlu-dist+cuda cuda_arch=70 ^magma+cuda cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" - extends: .lassen_build_and_test - -# ------------------------------------------------------------------------------ -# Benchmark jobs -# ------------------------------------------------------------------------------ - -lassen_gcc_cuda_bench: - parallel: - matrix: - - COMPILER_SPEC: gcc@8.3.1 - CUDA_SPEC: [cuda@11.8.0] - CALIPER_DIR: /usr/workspace/sundials/califiles - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double scheduler=lsf caliper-dir=${CALIPER_DIR} ~int64 +benchmarks+profiling+caliper+adiak+mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+adiak+cuda cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers" - extends: .lassen_build_and_bench - diff --git a/.gitlab/lassen-templates.yml b/.gitlab/lassen-templates.yml deleted file mode 100644 index 450a3134c2..0000000000 --- a/.gitlab/lassen-templates.yml +++ /dev/null @@ -1,56 +0,0 @@ -# ------------------------------------------------------------------------------ -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# ------------------------------------------------------------------------------ - -# ------------------------------------------------------------------------------ -# Tags and rules to run tests on Lassen -# ------------------------------------------------------------------------------ - -# Generic lassen build job, extending build script for IBM P9 systems -.lassen_build_and_test: - tags: - - shell - - lassen - extends: [.build_blueos_3_ppc64le_ib_script] - stage: l_build_and_test - needs: [] - artifacts: - paths: - - spack-*.txt - - build_*/* - when: always - rules: - # Don't run if... - - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF" || $BENCHMARK == "ON"' - when: never - # Default is to run if previous stage succeeded - - when: on_success - -.lassen_build_and_bench: - tags: - - shell - - lassen - extends: [.build_blueos_3_ppc64le_ib_bench] - stage: l_build_and_bench - needs: [] - artifacts: - paths: - - spack-*.txt - - build_*/* - - '*.cali' - when: always - expire_in: never - rules: - # Don't run if... - - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_LASSEN == "OFF" || $BENCHMARK == "OFF"' - when: never - # Default is to run if previous stage succeeded - - when: on_success diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml deleted file mode 100644 index a0c96755a6..0000000000 --- a/.gitlab/quartz-jobs.yml +++ /dev/null @@ -1,85 +0,0 @@ -# --------------------------------------------------------------- -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# --------------------------------------------------------------- - -# ------------------------------------------------------------------------------ -# Basic builds w/o TPLs -# ------------------------------------------------------------------------------ -### Builds without TPLs -# How to easily add +f2003 when int64 and double? - -# quartz_clang: -# parallel: -# matrix: -# - COMPILER_SPEC: clang@12.0.0 -# INDEX_SPEC: [~int64, +int64] -# PRECISION_SPEC: [double] -# variables: -# SPEC: "%${COMPILER_SPEC} ${INDEX_SPEC} precision=${PRECISION_SPEC}" -# extends: .quartz_build_and_test - -# quartz_gcc: -# parallel: -# matrix: -# - COMPILER_SPEC: gcc@10.3.1 -# INDEX_SPEC: [~int64, +int64] -# PRECISION_SPEC: [double] -# variables: -# SPEC: "%${COMPILER_SPEC} ${INDEX_SPEC} precision=${PRECISION_SPEC}" -# extends: .quartz_build_and_test - -# quartz_intel: -# parallel: -# matrix: -# - COMPILER_SPEC: intel@19.1.2 -# INDEX_SPEC: [~int64, +int64] -# PRECISION_SPEC: [double] -# variables: -# SPEC: "%${COMPILER_SPEC} ${INDEX_SPEC} precision=${PRECISION_SPEC}" -# extends: .quartz_build_and_test - -# ------------------------------------------------------------------------------ -# Builds with TPLs -# ------------------------------------------------------------------------------ -# +petsc ~hypre ~superlu-dist ~int64 ~hdf5 -# +superlu-dist == segfaults - -quartz_clang_tpls: - parallel: - matrix: - - COMPILER_SPEC: clang@14.0.6 - INDEX_SPEC: [~int64] - PRECISION_SPEC: [double] - variables: - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu +petsc ^suite-sparse@5.13.0 ^openblas" - extends: .quartz_build_and_test - -quartz_gcc_tpls: - parallel: - matrix: - - COMPILER_SPEC: gcc@10.3.1 - INDEX_SPEC: [~int64] - PRECISION_SPEC: [double] - variables: - # For some reason nvhpc gets picked up for lapack w/o ^openblas - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre +superlu-dist +lapack +klu +petsc ^suite-sparse@5.13.0 ^openblas" - extends: .quartz_build_and_test - -quartz_intel_tpls: - parallel: - matrix: - - COMPILER_SPEC: intel@19.1.2 - INDEX_SPEC: [~int64] - PRECISION_SPEC: [double] - variables: - # perl@5.32.0 needed until spack v0.20.0: https://github.com/spack/spack/pull/35666 - SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 ${INDEX_SPEC} precision=${PRECISION_SPEC} +mpi +openmp +hypre ~superlu-dist +lapack +klu ^suite-sparse@5.13.0 ^perl@5.32.0" - extends: .quartz_build_and_test diff --git a/.gitlab/quartz-templates.yml b/.gitlab/quartz-templates.yml deleted file mode 100644 index bfc33a3fd5..0000000000 --- a/.gitlab/quartz-templates.yml +++ /dev/null @@ -1,35 +0,0 @@ -# ------------------------------------------------------------------------------ -# SUNDIALS Copyright Start -# Copyright (c) 2002-2021, Lawrence Livermore National Security -# and Southern Methodist University. -# All rights reserved. -# -# See the top-level LICENSE and NOTICE files for details. -# -# SPDX-License-Identifier: BSD-3-Clause -# SUNDIALS Copyright End -# ------------------------------------------------------------------------------ - -# ------------------------------------------------------------------------------ -# Tags and rules to run tests on Quartz -# ------------------------------------------------------------------------------ - -# Generic quartz build job, extending build script -.quartz_build_and_test: - tags: - - shell - - quartz - extends: [.build_toss_3_x86_64_ib_script] - stage: q_build_and_test - needs: [] - artifacts: - paths: - - spack-*.txt - - build_*/* - when: always - rules: - # Don't run if... - - if: '$CI_COMMIT_BRANCH =~ /_lnone/ || $ON_QUARTZ == "OFF" || $BENCHMARK == "ON"' - when: never - # Default is to run if previous stage succeeded - - when: on_success diff --git a/.gitlab/radiuss-spack-configs b/.gitlab/radiuss-spack-configs index 1264a6db12..89c2074e46 160000 --- a/.gitlab/radiuss-spack-configs +++ b/.gitlab/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 1264a6db12f451153a5cb22a32a1b7ba2faa5fc2 +Subproject commit 89c2074e466d454c77ec17249cbdf8555d183388 diff --git a/.gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch b/.gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch deleted file mode 100644 index c9e1707857..0000000000 --- a/.gitlab/spack_packages/netlib-lapack/ibm-xl-3.9.1.patch +++ /dev/null @@ -1,108 +0,0 @@ -Fixes for IBM XL and Cray CCE builds: - -* Correct path to the fallback configuration used to handle mangling for - C++/Fortran compatibility (CCE, XL) - -* Change logic for detecting recursive fortran flags to (a) Include XL -(qrecur), and (b) Be explicit, since not every compiler will correctly reject -an incorrect option (ALL) - -NOTE: This patch has been accepted upstream -(see https://github.com/Reference-LAPACK/lapack/pull/621) - -############################################################################## - -diff -Naur a/CBLAS/CMakeLists.txt b/CBLAS/CMakeLists.txt ---- a/CBLAS/CMakeLists.txt 2021-03-25 12:25:15.000000000 -0600 -+++ b/CBLAS/CMakeLists.txt 2021-09-01 16:27:23.561355382 -0600 -@@ -11,9 +11,7 @@ - MACRO_NAMESPACE "F77_" - SYMBOL_NAMESPACE "F77_") - if(NOT FortranCInterface_GLOBAL_FOUND OR NOT FortranCInterface_MODULE_FOUND) -- message(WARNING "Reverting to pre-defined include/lapacke_mangling.h") -- configure_file(include/lapacke_mangling_with_flags.h.in -- ${LAPACK_BINARY_DIR}/include/lapacke_mangling.h) -+ message(WARNING "Reverting to pre-defined include/cblas_mangling.h") - configure_file(include/cblas_mangling_with_flags.h.in - ${LAPACK_BINARY_DIR}/include/cblas_mangling.h) - endif() -diff -Naur a/CMakeLists.txt b/CMakeLists.txt ---- a/CMakeLists.txt 2021-03-25 12:25:15.000000000 -0600 -+++ b/CMakeLists.txt 2021-09-02 09:49:18.070436958 -0600 -@@ -94,16 +94,22 @@ - - # Check if recursive flag exists - include(CheckFortranCompilerFlag) --check_fortran_compiler_flag("-recursive" _recursiveFlag) --check_fortran_compiler_flag("-frecursive" _frecursiveFlag) --check_fortran_compiler_flag("-Mrecursive" _MrecursiveFlag) -+if(CMAKE_Fortran_COMPILER_ID STREQUAL Flang) -+ check_fortran_compiler_flag("-Mrecursive" _MrecursiveFlag) -+elseif(CMAKE_Fortran_COMPILER_ID STREQUAL GNU) -+ check_fortran_compiler_flag("-frecursive" _frecursiveFlag) -+elseif(CMAKE_Fortran_COMPILER_ID STREQUAL Intel) -+ check_fortran_compiler_flag("-recursive" _recursiveFlag) -+elseif(CMAKE_Fortran_COMPILER_ID STREQUAL XL) -+ check_fortran_compiler_flag("-qrecur" _qrecurFlag) -+endif() - - # Add recursive flag --if(_recursiveFlag) -- string(REGEX MATCH "-recursive" output_test "${CMAKE_Fortran_FLAGS}") -+if(_MrecursiveFlag) -+ string(REGEX MATCH "-Mrecursive" output_test "${CMAKE_Fortran_FLAGS}") - if(NOT output_test) -- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -recursive" -- CACHE STRING "Recursive flag must be set" FORCE) -+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Mrecursive" -+ CACHE STRING "Recursive flag must be set" FORCE) - endif() - elseif(_frecursiveFlag) - string(REGEX MATCH "-frecursive" output_test "${CMAKE_Fortran_FLAGS}") -@@ -111,11 +117,17 @@ - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -frecursive" - CACHE STRING "Recursive flag must be set" FORCE) - endif() --elseif(_MrecursiveFlag) -- string(REGEX MATCH "-Mrecursive" output_test "${CMAKE_Fortran_FLAGS}") -+elseif(_recursiveFlag) -+ string(REGEX MATCH "-recursive" output_test "${CMAKE_Fortran_FLAGS}") - if(NOT output_test) -- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Mrecursive" -- CACHE STRING "Recursive flag must be set" FORCE) -+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -recursive" -+ CACHE STRING "Recursive flag must be set" FORCE) -+ endif() -+elseif(_qrecurFlag) -+ string(REGEX MATCH "-qrecur" output_test "${CMAKE_Fortran_FLAGS}") -+ if(NOT output_test) -+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qrecur" -+ CACHE STRING "Recursive flag must be set" FORCE) - endif() - endif() - -@@ -124,7 +136,7 @@ - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fp-model strict") - endif() - if(CMAKE_Fortran_COMPILER_ID STREQUAL XL) -- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict=none") -+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict") - endif() - # Delete libmtsk in linking sequence for Sun/Oracle Fortran Compiler. - # This library is not present in the Sun package SolarisStudio12.3-linux-x86-bin -diff -Naur a/INSTALL/make.inc.XLF b/INSTALL/make.inc.XLF ---- a/INSTALL/make.inc.XLF 2021-03-25 12:25:15.000000000 -0600 -+++ b/INSTALL/make.inc.XLF 2021-09-02 09:50:02.664646455 -0600 -@@ -14,10 +14,10 @@ - # the compiler options desired when NO OPTIMIZATION is selected. - # - FC = xlf --FFLAGS = -O3 -qfixed -qnosave -+FFLAGS = -O3 -qfixed -qnosave -qrecur - # For -O2, add -qstrict=none - FFLAGS_DRV = $(FFLAGS) --FFLAGS_NOOPT = -O0 -qfixed -qnosave -+FFLAGS_NOOPT = -O0 -qfixed -qnosave -qrecur - - # Define LDFLAGS to the desired linker options for your machine. - # diff --git a/.gitlab/spack_packages/netlib-lapack/ibm-xl.patch b/.gitlab/spack_packages/netlib-lapack/ibm-xl.patch deleted file mode 100644 index 52b5f19719..0000000000 --- a/.gitlab/spack_packages/netlib-lapack/ibm-xl.patch +++ /dev/null @@ -1,53 +0,0 @@ -Fixes for IBM XL and Cray CCE builds: - -* Avoid optimizations that would alter program semantics by changing the - qstrict activation threshold from O3 to O2 (XL) - -* Don't assume Fortran code is all in fixed source form; disable qfixed (XL) - -* Correct path to the fallback configuration used to handle mangling for - C++/Fortran compatibility (CCE, XL) -############################################################################## - ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -62,7 +62,7 @@ - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fp-model strict") - endif() - if("${CMAKE_Fortran_COMPILER}" MATCHES "xlf") -- set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict=none") -+ set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qnosave -qstrict") - endif() - # Delete libmtsk in linking sequence for Sun/Oracle Fortran Compiler. - # This library is not present in the Sun package SolarisStudio12.3-linux-x86-bin - ---- a/CMAKE/CheckLAPACKCompilerFlags.cmake -+++ b/CMAKE/CheckLAPACKCompilerFlags.cmake -@@ -43,12 +43,6 @@ - if( "${CMAKE_Fortran_FLAGS}" MATCHES "-qflttrap=[a-zA-Z:]:enable" ) - set( FPE_EXIT TRUE ) - endif() -- -- if( NOT ("${CMAKE_Fortran_FLAGS}" MATCHES "-qfixed") ) -- message( STATUS "Enabling fixed format F90/F95 with -qfixed" ) -- set( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qfixed" -- CACHE STRING "Flags for Fortran compiler." FORCE ) -- endif() - - # HP Fortran - elseif( CMAKE_Fortran_COMPILER_ID STREQUAL "HP" ) - ---- a/CBLAS/CMakeLists.txt -+++ b/CBLAS/CMakeLists.txt -@@ -12,8 +12,8 @@ - SYMBOL_NAMESPACE "F77_") - if(NOT FortranCInterface_GLOBAL_FOUND OR NOT FortranCInterface_MODULE_FOUND) - message(WARNING "Reverting to pre-defined include/lapacke_mangling.h") -- configure_file(include/lapacke_mangling_with_flags.h.in -- ${LAPACK_BINARY_DIR}/include/lapacke_mangling.h) -+ configure_file(include/cblas_mangling_with_flags.h.in -+ ${LAPACK_BINARY_DIR}/include/cblas_mangling.h) - endif() - - include_directories(include ${LAPACK_BINARY_DIR}/include) - diff --git a/.gitlab/spack_packages/netlib-lapack/package.py b/.gitlab/spack_packages/netlib-lapack/package.py deleted file mode 100644 index cb1832d061..0000000000 --- a/.gitlab/spack_packages/netlib-lapack/package.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -import spack.build_systems.cmake -from spack.package import * - - -class NetlibLapack(CMakePackage): - """LAPACK version 3.X is a comprehensive FORTRAN library that does - linear algebra operations including matrix inversions, least squared - solutions to linear sets of equations, eigenvector analysis, singular - value decomposition, etc. It is a very comprehensive and reputable - package that has found extensive use in the scientific community. - - """ - - homepage = "https://www.netlib.org/lapack/" - url = "https://www.netlib.org/lapack/lapack-3.5.0.tgz" - tags = ["windows"] - - version( - "3.10.1", - sha256="cd005cd021f144d7d5f7f33c943942db9f03a28d110d6a3b80d718a295f7f714", - url="https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.10.1.tar.gz", - ) - version( - "3.10.0", - sha256="328c1bea493a32cac5257d84157dc686cc3ab0b004e2bea22044e0a59f6f8a19", - url="https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.10.0.tar.gz", - ) - version( - "3.9.1", - sha256="d0085d2caf997ff39299c05d4bacb6f3d27001d25a4cc613d48c1f352b73e7e0", - url="https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.9.1.tar.gz", - ) - version( - "3.9.0", - sha256="106087f1bb5f46afdfba7f569d0cbe23dacb9a07cd24733765a0e89dbe1ad573", - url="https://github.com/Reference-LAPACK/lapack/archive/v3.9.0.tar.gz", - ) - version( - "3.8.0", - sha256="deb22cc4a6120bff72621155a9917f485f96ef8319ac074a7afbc68aab88bcf6", - url="https://www.netlib.org/lapack/lapack-3.8.0.tar.gz", - ) - version("3.7.1", sha256="f6c53fd9f56932f3ddb3d5e24c1c07e4cd9b3b08e7f89de9c867125eecc9a1c8") - version("3.7.0", sha256="ed967e4307e986474ab02eb810eed1d1adc73f5e1e3bc78fb009f6fe766db3be") - version("3.6.1", sha256="888a50d787a9d828074db581c80b2d22bdb91435a673b1bf6cd6eb51aa50d1de") - version("3.6.0", sha256="a9a0082c918fe14e377bbd570057616768dca76cbdc713457d8199aaa233ffc3") - version("3.5.0", sha256="9ad8f0d3f3fb5521db49f2dd716463b8fb2b6bc9dc386a9956b8c6144f726352") - version("3.4.2", sha256="60a65daaf16ec315034675942618a2230521ea7adf85eea788ee54841072faf0") - version("3.4.1", sha256="93b910f94f6091a2e71b59809c4db4a14655db527cfc5821ade2e8c8ab75380f") - version("3.4.0", sha256="a7139ef97004d0e3c4c30f1c52d508fd7ae84b5fbaf0dd8e792c167dc306c3e9") - version("3.3.1", sha256="56821ab51c29369a34e5085728f92c549a9aa926f26acf7eeac87b61eed329e4") - - # netlib-lapack is the reference implementation of LAPACK - for ver in [ - "3.10.1", - "3.10.0", - "3.9.1", - "3.9.0", - "3.8.0", - "3.7.1", - "3.7.0", - "3.6.1", - "3.6.0", - "3.5.0", - "3.4.2", - "3.4.1", - "3.4.0", - "3.3.1", - ]: - provides("lapack@" + ver, when="@" + ver) - - variant("shared", default=True, description="Build shared library version") - variant("external-blas", default=False, description="Build lapack with an external blas") - - variant("lapacke", default=True, description="Activates the build of the LAPACKE C interface") - variant("xblas", default=False, description="Builds extended precision routines using XBLAS") - - # Fixes for IBM XL and Cray CCE builds: - # Avoid optimizations that alter program semantics - # Don't assume fixed source form for Fortran - # Correct path to mangling config - patch("ibm-xl.patch", when="@3.7:3.8 %xl") - patch("ibm-xl.patch", when="@3.7:3.8 %xl_r") - patch("ibm-xl.patch", when="@3.7:3.8 %cce@9:") - - # https://github.com/Reference-LAPACK/lapack/pull/621 - # Fixes for IBM XL and Cray CCE builds: - # Correct path to mangling config - # Fix logic for detecting recursive Fortran flags - patch("ibm-xl-3.9.1.patch", when="@3.9.1 %xl") - patch("ibm-xl-3.9.1.patch", when="@3.9.1 %xl_r") - patch("ibm-xl-3.9.1.patch", when="@3.9.1 %cce@13:") - - # https://github.com/Reference-LAPACK/lapack/issues/228 - patch("undefined_declarations.patch", when="@3.8.0:3.8") - - # https://github.com/Reference-LAPACK/lapack/pull/268 - patch("testing.patch", when="@3.7.0:3.8") - - # virtual dependency - provides("blas", when="~external-blas") - provides("lapack") - - depends_on("blas", when="+external-blas") - depends_on("netlib-xblas+fortran+plain_blas", when="+xblas") - depends_on("python@2.7:", type="test") - - # We need to run every phase twice in order to get static and shared - # versions of the libraries. When ~shared, we run the default - # implementations of the CMakePackage's phases and get only one building - # directory 'spack-build-static' with -DBUILD_SHARED_LIBS:BOOL=OFF (see - # implementations of self.build_directory and self.cmake_args() below). - # When +shared, we run the overridden methods for the phases, each - # running the default implementation twice with different values for - # self._building_shared. As a result, we get two building directories: - # 'spack-build-static' with -DBUILD_SHARED_LIBS:BOOL=OFF and - # 'spack-build-shared' with -DBUILD_SHARED_LIBS:BOOL=ON. - _building_shared = False - - def patch(self): - # Fix cblas CMakeLists.txt -- has wrong case for subdirectory name. - if self.spec.satisfies("@3.6.0:"): - filter_file( - "${CMAKE_CURRENT_SOURCE_DIR}/CMAKE/", - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/", - "CBLAS/CMakeLists.txt", - string=True, - ) - - # Remove duplicate header file that gets generated during CMake shared - # builds: https://github.com/Reference-LAPACK/lapack/issues/583 - if self.spec.satisfies("platform=windows @0:3.9.1"): - force_remove("LAPACKE/include/lapacke_mangling.h") - - @property - def blas_libs(self): - shared = True if "+shared" in self.spec else False - query_parameters = self.spec.last_query.extra_parameters - query2libraries = { - tuple(): ["libblas"], - ("c", "fortran"): ["libcblas", "libblas"], - ("c",): ["libcblas"], - ("fortran",): ["libblas"], - } - key = tuple(sorted(query_parameters)) - libraries = query2libraries[key] - return find_libraries(libraries, root=self.prefix, shared=shared, recursive=True) - - @property - def lapack_libs(self): - shared = True if "+shared" in self.spec else False - query_parameters = self.spec.last_query.extra_parameters - query2libraries = { - tuple(): ["liblapack"], - ("c", "fortran"): ["liblapacke", "liblapack"], - ("c",): ["liblapacke"], - ("fortran",): ["liblapack"], - } - key = tuple(sorted(query_parameters)) - libraries = query2libraries[key] - return find_libraries(libraries, root=self.prefix, shared=shared, recursive=True) - - @property - def headers(self): - include_dir = self.spec.prefix.include - cblas_h = join_path(include_dir, "cblas.h") - lapacke_h = join_path(include_dir, "lapacke.h") - return HeaderList([cblas_h, lapacke_h]) - - -class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder): - def cmake_args(self): - args = [ - self.define_from_variant("BUILD_SHARED_LIBS", "shared"), - self.define_from_variant("LAPACKE", "lapacke"), - self.define_from_variant("LAPACKE_WITH_TMG", "lapacke"), - self.define("CBLAS", self.spec.satisfies("@3.6.0:")), - ] - - if self.spec.satisfies("%intel"): - # Intel compiler finds serious syntax issues when trying to - # build CBLAS and LapackE - args.extend([self.define("CBLAS", False), self.define("LAPACKE", False)]) - - if self.spec.satisfies("%xl") or self.spec.satisfies("%xl_r"): - # use F77 compiler if IBM XL - args.extend( - [ - self.define("CMAKE_Fortran_COMPILER", self.pkg.compiler.f77), - self.define( - "CMAKE_Fortran_FLAGS", - " ".join(self.spec.compiler_flags["fflags"]) + " -O3 -qnohot", - ), - ] - ) - - # deprecated routines are commonly needed by, for example, suitesparse - # Note that OpenBLAS spack is built with deprecated routines - args.append(self.define("BUILD_DEPRECATED", True)) - - if self.spec.satisfies("+external-blas"): - args.extend( - [ - self.define("USE_OPTIMIZED_BLAS", True), - self.define("BLAS_LIBRARIES:PATH", self.spec["blas"].libs.joined(";")), - ] - ) - - if self.spec.satisfies("+xblas"): - args.extend( - [ - self.define("XBLAS_INCLUDE_DIR", self.spec["netlib-xblas"].prefix.include), - self.define("XBLAS_LIBRARY", self.spec["netlib-xblas"].libs.joined(";")), - ] - ) - - args.append(self.define("BUILD_TESTING", self.pkg.run_tests)) - - return args diff --git a/.gitlab/spack_packages/netlib-lapack/testing.patch b/.gitlab/spack_packages/netlib-lapack/testing.patch deleted file mode 100644 index fce18548c4..0000000000 --- a/.gitlab/spack_packages/netlib-lapack/testing.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/TESTING/LIN/alahd.f b/TESTING/LIN/alahd.f -index 8f4cd58d..6a4946e0 100644 ---- a/TESTING/LIN/alahd.f -+++ b/TESTING/LIN/alahd.f -@@ -1036,7 +1036,7 @@ - 9929 FORMAT( ' Test ratios (1-3: ', A1, 'TZRZF):' ) - 9920 FORMAT( 3X, ' 7-10: same as 3-6', 3X, ' 11-14: same as 3-6' ) - 9921 FORMAT( ' Test ratios:', / ' (1-2: ', A1, 'GELS, 3-6: ', A1, -- $ 'GELSY, 7-10: ', A1, 'GELSS, 11-14: ', A1, 'GELSD, 15-16: ' -+ $ 'GELSY, 7-10: ', A1, 'GELSS, 11-14: ', A1, 'GELSD, 15-16: ', - $ A1, 'GETSLS)') - 9928 FORMAT( 7X, 'where ALPHA = ( 1 + SQRT( 17 ) ) / 8' ) - 9927 FORMAT( 3X, I2, ': ABS( Largest element in L )', / 12X, diff --git a/.gitlab/spack_packages/netlib-lapack/undefined_declarations.patch b/.gitlab/spack_packages/netlib-lapack/undefined_declarations.patch deleted file mode 100644 index 9dac2562f7..0000000000 --- a/.gitlab/spack_packages/netlib-lapack/undefined_declarations.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/SRC/dsytrf_aa_2stage.f b/SRC/dsytrf_aa_2stage.f -index 2991305..f5f06cc 100644 ---- a/SRC/dsytrf_aa_2stage.f -+++ b/SRC/dsytrf_aa_2stage.f -@@ -191,7 +191,7 @@ - EXTERNAL LSAME, ILAENV - * .. - * .. External Subroutines .. -- EXTERNAL XERBLA, DCOPY, DLACGV, DLACPY, -+ EXTERNAL XERBLA, DCOPY, DLACPY, - $ DLASET, DGBTRF, DGEMM, DGETRF, - $ DSYGST, DSWAP, DTRSM - * .. -diff --git a/SRC/ssytrf_aa_2stage.f b/SRC/ssytrf_aa_2stage.f -index be6809d..a929749 100644 ---- a/SRC/ssytrf_aa_2stage.f -+++ b/SRC/ssytrf_aa_2stage.f -@@ -191,7 +191,7 @@ - EXTERNAL LSAME, ILAENV - * .. - * .. External Subroutines .. -- EXTERNAL XERBLA, SCOPY, SLACGV, SLACPY, -+ EXTERNAL XERBLA, SCOPY, SLACPY, - $ SLASET, SGBTRF, SGEMM, SGETRF, - $ SSYGST, SSWAP, STRSM - * .. diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml new file mode 100644 index 0000000000..879dcc7be6 --- /dev/null +++ b/.gitlab/subscribed-pipelines.yml @@ -0,0 +1,77 @@ +############################################################################### +# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS +# project contributors. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# SPDX-License-Identifier: (MIT) +############################################################################### +# Modifications to this file have been made under the folllowing copyright. +# ----------------------------------------------------------------------------- +# SUNDIALS Copyright Start +# Copyright (c) 2002-2024, Lawrence Livermore National Security +# and Southern Methodist University. +# All rights reserved. +# +# See the top-level LICENSE and NOTICE files for details. +# +# SPDX-License-Identifier: BSD-3-Clause +# SUNDIALS Copyright End +# ----------------------------------------------------------------------------- + +# The template job to test whether a machine is up. +# Expects CI_MACHINE defined to machine name. +.machine-check: + stage: prerequisites + tags: [shell, oslic] + variables: + GIT_STRATEGY: none + script: + - | + if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]] + then + echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m" + curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \ + --header 'Content-Type: application/json' \ + --header "authorization: Bearer ${GITHUB_TOKEN}" \ + --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }" + exit 1 + fi + +### +# Trigger a build-and-test pipeline for a machine. +# Comment the jobs for machines you don’t need. +### + +# DANE +dane-up-check: + variables: + CI_MACHINE: "dane" + extends: [.machine-check] + +dane-build-and-test: + variables: + CI_MACHINE: "dane" + needs: [dane-up-check] + extends: [.build-and-test] + +# TIOGA +tioga-up-check: + variables: + CI_MACHINE: "tioga" + extends: [.machine-check] + +tioga-build-and-test: + variables: + CI_MACHINE: "tioga" + needs: [tioga-up-check] + extends: [.build-and-test] + diff --git a/.gitlab/uberenv b/.gitlab/uberenv index 0d00dc8e19..ec6fd144c8 160000 --- a/.gitlab/uberenv +++ b/.gitlab/uberenv @@ -1 +1 @@ -Subproject commit 0d00dc8e19a889ba07ae433590b87533c4b5b3da +Subproject commit ec6fd144c814ce30a6487db9fb8294def606afda diff --git a/.gitmodules b/.gitmodules index 659855b75f..e915f32aa6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ url = https://github.com/sundials-codes/radiuss-spack-configs.git [submodule "uberenv"] path = .gitlab/uberenv - url = https://github.com/LLNL/uberenv.git + url = https://github.com/sundials-codes/uberenv.git [submodule "test/answers"] path = test/answers url = https://github.com/sundials-codes/answers.git diff --git a/.uberenv_config.json b/.uberenv_config.json index 67618cca2f..3264fe5b05 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -4,8 +4,7 @@ "package_final_phase": "initconfig", "package_source_dir": "../..", "spack_url": "https://github.com/spack/spack", - "spack_commit": "5e0d2107348eed6cbe6deca43a30f5b06c5e40af", "spack_activate": {}, "spack_configs_path": ".gitlab/radiuss-spack-configs", - "spack_packages_path": ".gitlab/spack_packages" -} + "spack_packages_path": "scripts/spack/packages" +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index eee9a428a5..e4fd3ff71f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,178 @@ # SUNDIALS Changelog +## Changes to SUNDIALS in release 7.2.0 + +### Major Features + +Added a time-stepping module to ARKODE for low storage Runge--Kutta methods, +LSRKStep. This currently supports five explicit low-storage methods: the +second-order Runge--Kutta--Chebyshev and Runge--Kutta--Legendre methods, and the +second- through fourth-order optimal strong stability preserving Runge--Kutta +methods. All methods include embeddings for temporal adaptivity. + +Added an operator splitting module, SplittingStep, and forcing method module, +ForcingStep, to ARKODE. These modules support a broad range of operator-split +time integration methods for multiphysics applications. + +Added support for multirate time step adaptivity controllers, based on the +recently introduced `SUNAdaptController` base class, to ARKODE's MRIStep module. +As a part of this, we added embeddings for existing MRI-GARK methods, as well as +support for embedded MERK and IMEX-MRI-SR methods. Added new default MRI methods +for temporally adaptive versus fixed-step runs. + +### New Features and Enhancements + +#### Logging + +The information level logging output in ARKODE, CVODE(S), and IDA(S) has been +updated to be more uniform across the packages and a new `tools` directory has +been added with a Python module, `suntools`, containing utilities for parsing +logging output. The Python utilities for parsing CSV output have been relocated +from the `scripts` directory to the Python module. + +#### SUNStepper + +Added the `SUNStepper` base class to represent a generic solution procedure for +IVPs. This is used by the SplittingStep and ForcingStep modules of ARKODE. A +SUNStepper can be created from an ARKODE memory block with the new function +`ARKodeCreateSUNStepper`. To enable interoperability with `MRIStepInnerStepper`, +the function `MRIStepInnerStepper_CreateFromSUNStepper` was added. + +#### ARKODE + +Added functionality to ARKODE to accumulate a temporal error estimate over +multiple time steps. See the routines `ARKodeSetAccumulatedErrorType`, +`ARKodeResetAccumulatedError`, and `ARKodeGetAccumulatedError` for details. + +Added the `ARKodeSetStepDirection` and `ARKodeGetStepDirection` functions to +change and query the direction of integration. + +Added the function `MRIStepGetNumInnerStepperFails` to retrieve the number of +recoverable failures reported by the MRIStepInnerStepper. + +Added a utility routine to wrap any valid ARKODE integrator for use as an +MRIStep inner stepper object, `ARKodeCreateMRIStepInnerStepper`. + +The following DIRK schemes now have coefficients accurate to quad precision: +* `ARKODE_BILLINGTON_3_3_2` +* `ARKODE_KVAERNO_4_2_3` +* `ARKODE_CASH_5_2_4` +* `ARKODE_CASH_5_3_4` +* `ARKODE_KVAERNO_5_3_4` +* `ARKODE_KVAERNO_7_4_5` + +#### CMake + +The default value of `CMAKE_CUDA_ARCHITECTURES` is no longer set to `70` and is +now determined automatically by CMake. The previous default was only valid for +Volta GPUs while the automatically selected value will vary across compilers and +compiler versions. As such, users are encouraged to override this value with the +architecture for their system. + +The build system has been updated to utilize the CMake LAPACK imported target +which should ease building SUNDIALS with LAPACK libraries that require setting +specific linker flags e.g., MKL. + +#### Third Party Libraries + +The Trilinos Teptra NVector interface has been updated to utilize CMake imported +targets added in Trilinos 14 to improve support for different Kokkos backends +with Trilinos. As such, Trilinos 14 or newer is required and the +`Trilinos_INTERFACE_*` CMake options have been removed. + +Example programs using *hypre* have been updated to support v2.20 and newer. + +### Bug Fixes + +#### CMake + +Fixed a CMake bug regarding usage of missing "print_warning" macro that was only +triggered when the deprecated `CUDA_ARCH` option was used. + +Fixed a CMake configuration issue related to aliasing an `ALIAS` target when +using `ENABLE_KLU=ON` in combination with a static-only build of SuiteSparse. + +Fixed a CMake issue which caused third-party CMake variables to be unset. Users +may see more options in the CMake GUI now as a result of the fix. See details +in GitHub Issue [#538](https://github.com/LLNL/sundials/issues/538). + +#### NVector + +Fixed a build failure with the SYCL NVector when using Intel oneAPI 2025.0 +compilers. See GitHub Issue [#596](https://github.com/LLNL/sundials/issues/596). + +Fixed compilation errors when building the Trilinos Teptra NVector with CUDA +support. + +#### SUNMatrix + +Fixed a [bug](https://github.com/LLNL/sundials/issues/581) in the sparse matrix +implementation of `SUNMatScaleAddI` which caused out of bounds writes unless +`indexvals` were in ascending order for each row/column. + +#### SUNLinearSolver + +Fixed a bug in the SPTFQMR linear solver where recoverable preconditioner errors +were reported as unrecoverable. + +#### ARKODE + +Fixed `ARKodeResize` not using the default `hscale` when an argument of `0` was +provided. + +Fixed a memory leak that could occur if ``ARKodeSetDefaults`` is called +repeatedly. + +Fixed the loading of ARKStep's default first order explicit method. + +Fixed loading the default IMEX-MRI method if `ARKodeSetOrder` is used to specify +a third or fourth order method. Previously, the default second order method was +loaded in both cases. + +Fixed potential memory leaks and out of bounds array accesses that could occur +in the ARKODE Lagrange interpolation module when changing the method order or +polynomial degree after re-initializing an integrator. + +Fixed a bug in ARKODE when enabling rootfinding with fixed step sizes and the +initial value of the rootfinding function is zero. In this case, uninitialized +right-hand side data was used to compute a state value near the initial +condition to determine if any rootfinding functions are initially active. + +Fixed a bug in MRIStep where the data supplied to the Hermite interpolation +module did not include contributions from the fast right-hand side +function. With this fix, users will see one additional fast right-hand side +function evaluation per slow step with the Hermite interpolation option. + +Fixed a bug in SPRKStep when using compensated summations where the error vector +was not initialized to zero. + +#### CVODE(S) + +Fixed a bug where `CVodeSetProjFailEta` would ignore the `eta` parameter. + +#### Fortran Interfaces + +Fixed a bug in the 32-bit ``sunindextype`` Fortran interfaces to +``N_VGetSubvectorArrayPointer_ManyVector``, +``N_VGetSubvectorArrayPointer_MPIManyVector``, ``SUNBandMatrix_Column`` and +``SUNDenseMatrix_Column`` where 64-bit ``sunindextype`` interface functions were +used. + +### Deprecation Notices + +Deprecated the ARKStep-specific utility routine for wrapping an ARKStep instance +as an MRIStep inner stepper object, `ARKStepCreateMRIStepInnerStepper`. Use +`ARKodeCreateMRIStepInnerStepper` instead. + +The ARKODE stepper specific functions to retrieve the number of right-hand side +function evaluations have been deprecated. Use `ARKodeGetNumRhsEvals` instead. + ## Changes to SUNDIALS in release 7.1.1 ### Bug Fixes -Fixed a [bug](https://github.com/LLNL/sundials/pull/523) in v7.1.0 with the SYCL N_Vector `N_VSpace` function. +Fixed a [bug](https://github.com/LLNL/sundials/pull/523) in v7.1.0 with the SYCL +N_Vector `N_VSpace` function. ## Changes to SUNDIALS in release 7.1.0 @@ -135,7 +303,7 @@ ARKODE-wide equivalent, instructions have been added to the user guide for how to retain the current functionality using other user-callable functions. The unsupported implementations of `N_VGetArrayPointer` and `N_VSetArrayPointer` -for the *hypre* and PETSc vectors are now deprecated. Users should access the +for the *hypre* and PETSc vectors are now deprecated. Users should access the underlying wrapped external library vector objects instead with `N_VGetVector_ParHyp` and `N_VGetVector_Petsc`, respectively. @@ -442,9 +610,9 @@ ARKODE. ### New Features Updated CVODE, CVODES and ARKODE default behavior when returning the solution when -the internal time has reached a user-specified stop time. Previously, the output +the internal time has reached a user-specified stop time. Previously, the output solution was interpolated to the value of `tstop`; the default is now to copy the -internal solution vector. Users who wish to revert to interpolation may call a new +internal solution vector. Users who wish to revert to interpolation may call a new routine `CVodeSetInterpolateStopTime`, `ARKStepSetInterpolateStopTime`, `ERKStepSetInterpolateStopTime`, or `MRIStepSetInterpolateStopTime`. @@ -764,7 +932,7 @@ instead. * `SUNLinSolSetInfoFile_SPBCGS` * `SUNLinSolSetPrintLevel_SPBCGS` -The `SUNLinSolSetInfoFile_*` and `SUNNonlinSolSetInfoFile_*` family of +The `SUNLinSolSetInfoFile_*` and `SUNNonlinSolSetInfoFile_*` family of functions are now enabled by setting the CMake option `SUNDIALS_LOGGING_LEVEL` to a value `>= 3`. @@ -925,7 +1093,7 @@ can be enabled with the CMake option `SUNDIALS_BUILD_WITH_PROFILING`. A built-in profiler will be used by default, but the [Caliper](https://github.com/LLNL/Caliper) library can also be used instead with the CMake option `ENABLE_CALIPER`. See the documentation section on profiling -for more details. **WARNING**: Profiling will impact performance, and should be +for more details. **WARNING**: Profiling will impact performance, and should be enabled judiciously. #### IMEX MRI Methods and MRIStepInnerStepper Object @@ -1260,9 +1428,9 @@ use the `SUNLinSolNewEmpty` constructor will, at a minimum, need set the to leverage this new set function to remove one dot product per solve. The time integrator packages (ARKODE, CVODE(S), and IDA(S)) all now support a -new "matrix-embedded" SUNLinearSolver type. This type supports user-supplied +new "matrix-embedded" SUNLinearSolver type. This type supports user-supplied SUNLinearSolver implementations that set up and solve the specified linear -system at each linear solve call. Any matrix-related data structures are held +system at each linear solve call. Any matrix-related data structures are held internally to the linear solver itself, and are not provided by the SUNDIALS package. @@ -1637,7 +1805,7 @@ should be used instead. Added support for a user-supplied function to update the prediction for each implicit stage solution in ARKStep. If supplied, this routine will be called *after* any existing ARKStep predictor algorithm completes, so that the -predictor may be modified by the user as desired. The new user-supplied routine +predictor may be modified by the user as desired. The new user-supplied routine has type `ARKStepStagePredictFn`, and may be set by calling `ARKStepSetStagePredictFn`. @@ -1776,7 +1944,7 @@ update their code to set the corresponding `ops` structure member, `matvecsetup`, to `NULL`. The generic SUNMatrix API now defines error codes to be returned by SUNMatrix -operations. Operations which return an integer flag indiciating success/failure +operations. Operations which return an integer flag indicating success/failure may return different values than previously. A new SUNMatrix (and SUNLinearSolver) implementation was added to facilitate @@ -1835,7 +2003,7 @@ The inputs values passed to the first two inputs of the `SUNNonlinSolSolve` function in the `SUNNonlinearSolver` have been changed to be the predicted state and the initial guess for the correction to that state. Additionally, the definitions of `SUNNonlinSolLSetupFn` and `SUNNonlinSolLSolveFn` in the -SUNNonlinearSolver API have been updated to remove unused input parameters. For +SUNNonlinearSolver API have been updated to remove unused input parameters. For more information on the nonlinear system formulation and the API functions see the `SUNNonlinearSolver` chapter in the user guides. @@ -1888,7 +2056,7 @@ function signatures have been changed including MRIStepCreate which now takes an ARKStep memory structure for the fast integration as an input. The reinitialization functions `ERKStepReInit`, `ARKStepReInit`, and -`MRIStepReInit` have been updated to retain the minimum and maxiumum step +`MRIStepReInit` have been updated to retain the minimum and maximum step size values from before reinitialization rather than resetting them to the default values. @@ -1913,7 +2081,7 @@ being built. Fixed a memory leak in the PETSc `N_Vector` clone function. -Fixed a memeory leak in the ARKODE, CVODE, and IDA F77 interfaces when not using +Fixed a memory leak in the ARKODE, CVODE, and IDA F77 interfaces when not using the default nonlinear solver. Fixed a bug in the ARKStep time-stepping module in ARKODE that would result in @@ -2625,7 +2793,7 @@ with sparse direct solvers. #### KINSOL -The Picard iteration return was chanegd to always return the newest iterate upon +The Picard iteration return was changed to always return the newest iterate upon success. A minor bug in the line search was fixed to prevent an infinite loop when the diff --git a/CITATIONS.md b/CITATIONS.md index 9e93aa7ced..78a67ab733 100644 --- a/CITATIONS.md +++ b/CITATIONS.md @@ -69,7 +69,7 @@ they are using rather than the combined SUNDIALS online guide: author = {Daniel R. Reynolds and David J. Gardner and Carol S. Woodward and Cody J. Balos}, title = {User Documentation for ARKODE}, year = {2024}, - note = {v6.1.1} + note = {v6.2.0} } ``` @@ -78,7 +78,7 @@ they are using rather than the combined SUNDIALS online guide: author = {Alan C. Hindmarsh and Radu Serban and Cody J. Balos and David J. Gardner and Daniel R. Reynolds and Carol S. Woodward}, title = {User Documentation for CVODE}, year = {2024}, - note = {v7.1.1} + note = {v7.2.0} } ``` @@ -87,7 +87,7 @@ they are using rather than the combined SUNDIALS online guide: author = {Alan C. Hindmarsh and Radu Serban and Cody J. Balos and David J. Gardner and Daniel R. Reynolds and Carol S. Woodward}, title = {User Documentation for CVODES}, year = {2024}, - note = {v7.1.1} + note = {v7.2.0} } ``` @@ -96,7 +96,7 @@ they are using rather than the combined SUNDIALS online guide: author = {Alan C. Hindmarsh and Radu Serban and Cody J. Balos and David J. Gardner and Daniel R. Reynolds and Carol S. Woodward}, title = {User Documentation for IDA}, year = {2024}, - note = {v7.1.1} + note = {v7.2.0} } ``` @@ -105,7 +105,7 @@ they are using rather than the combined SUNDIALS online guide: author = {Radu Serban and Cosmin Petra and Alan C. Hindmarsh and Cody J. Balos and David J. Gardner and Daniel R. Reynolds and Carol S. Woodward}, title = {User Documentation for IDAS}, year = {2024}, - note = {v6.1.1} + note = {v6.2.0} } ``` @@ -114,6 +114,6 @@ they are using rather than the combined SUNDIALS online guide: author = {Alan C. Hindmarsh and Radu Serban and Cody J. Balos and David J. Gardner and Daniel R. Reynolds and Carol S. Woodward}, title = {User Documentation for KINSOL}, year = {2024}, - note = {v7.1.1} + note = {v7.2.0} } ``` diff --git a/CMakeLists.txt b/CMakeLists.txt index b5cca95afd..9a1aec9040 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,17 +26,14 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") cmake_policy(SET CMP0135 NEW) endif() -# Project SUNDIALS (initially only C supported) -# sets PROJECT_SOURCE_DIR and PROJECT_BINARY_DIR variables. +# Project SUNDIALS (initially only C supported) sets PROJECT_SOURCE_DIR and +# PROJECT_BINARY_DIR variables. project(SUNDIALS C) # Specify the location of additional CMAKE modules set(CMAKE_MODULE_PATH - ${CMAKE_MODULE_PATH} - ${PROJECT_SOURCE_DIR}/cmake - ${PROJECT_SOURCE_DIR}/cmake/macros - ${PROJECT_SOURCE_DIR}/cmake/tpl - ) + ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake + ${PROJECT_SOURCE_DIR}/cmake/macros ${PROJECT_SOURCE_DIR}/cmake/tpl) # MACRO definitions include(SundialsCMakeMacros) @@ -44,11 +41,10 @@ include(CMakePrintHelpers) include(CheckCSourceCompiles) include(FindPackageHandleStandardArgs) - # Set some variables with info on the SUNDIALS project set(PACKAGE_BUGREPORT "sundials-users@llnl.gov") set(PACKAGE_NAME "SUNDIALS") -set(PACKAGE_STRING "SUNDIALS 7.1.1") +set(PACKAGE_STRING "SUNDIALS 7.2.0") set(PACKAGE_TARNAME "sundials") # Set SUNDIALS version numbers @@ -57,8 +53,8 @@ message(STATUS "SUNDIALS_GIT_VERSION: ${SUNDIALS_GIT_VERSION}") # (use "" for the version label if none is needed) set(PACKAGE_VERSION_MAJOR "7") -set(PACKAGE_VERSION_MINOR "1") -set(PACKAGE_VERSION_PATCH "1") +set(PACKAGE_VERSION_MINOR "2") +set(PACKAGE_VERSION_PATCH "0") set(PACKAGE_VERSION_LABEL "") if(PACKAGE_VERSION_LABEL) @@ -73,37 +69,37 @@ endif() # Specify the VERSION and SOVERSION for shared libraries -set(arkodelib_VERSION "6.1.1") +set(arkodelib_VERSION "6.2.0") set(arkodelib_SOVERSION "6") -set(cvodelib_VERSION "7.1.1") +set(cvodelib_VERSION "7.2.0") set(cvodelib_SOVERSION "7") -set(cvodeslib_VERSION "7.1.1") +set(cvodeslib_VERSION "7.2.0") set(cvodeslib_SOVERSION "7") -set(idalib_VERSION "7.1.1") +set(idalib_VERSION "7.2.0") set(idalib_SOVERSION "7") -set(idaslib_VERSION "6.1.1") +set(idaslib_VERSION "6.2.0") set(idaslib_SOVERSION "6") -set(kinsollib_VERSION "7.1.1") +set(kinsollib_VERSION "7.2.0") set(kinsollib_SOVERSION "7") set(cpodeslib_VERSION "0.0.0") set(cpodeslib_SOVERSION "0") -set(nveclib_VERSION "7.1.1") +set(nveclib_VERSION "7.2.0") set(nveclib_SOVERSION "7") -set(sunmatrixlib_VERSION "5.1.1") +set(sunmatrixlib_VERSION "5.2.0") set(sunmatrixlib_SOVERSION "5") -set(sunlinsollib_VERSION "5.1.1") +set(sunlinsollib_VERSION "5.2.0") set(sunlinsollib_SOVERSION "5") -set(sunnonlinsollib_VERSION "4.1.1") +set(sunnonlinsollib_VERSION "4.2.0") set(sunnonlinsollib_SOVERSION "4") set(sundialslib_VERSION @@ -123,9 +119,9 @@ endif() # Organize targets into folders when using an IDE set_property(GLOBAL PROPERTY USE_FOLDERS ON) -# Get correct build paths automatically, but expose LIBDIR and -# INCLUDEDIR as a regular cache variable so that a user can more -# easily see what they were set to by GNUInstallDirs. +# Get correct build paths automatically, but expose LIBDIR and INCLUDEDIR as a +# regular cache variable so that a user can more easily see what they were set +# to by GNUInstallDirs. include(GNUInstallDirs) mark_as_advanced(CLEAR CMAKE_INSTALL_LIBDIR) mark_as_advanced(CLEAR CMAKE_INSTALL_INCLUDEDIR) @@ -133,23 +129,28 @@ mark_as_advanced(CLEAR CMAKE_INSTALL_INCLUDEDIR) # Suffixes to use for static and shared targets. set(_STATIC_LIB_SUFFIX "_static" - CACHE INTERNAL "" FORCE -) + CACHE INTERNAL "" FORCE) set(_SHARED_LIB_SUFFIX "_shared" - CACHE INTERNAL "" FORCE -) + CACHE INTERNAL "" FORCE) # A list of all the alias targets created. -set(_SUNDIALS_ALIAS_TARGETS "" +set(_SUNDIALS_ALIAS_TARGETS + "" CACHE INTERNAL "" FORCE) # We default to release builds set(_DEFAULT_CMAKE_BUILD_TYPE RelWithDebInfo) if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Building SUNDIALS in '${_DEFAULT_CMAKE_BUILD_TYPE}' mode as CMAKE_BUILD_TYPE was not specified.") - set(CMAKE_BUILD_TYPE "${_DEFAULT_CMAKE_BUILD_TYPE}" CACHE STRING "Choose the type of build." FORCE) - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") + message( + STATUS + "Building SUNDIALS in '${_DEFAULT_CMAKE_BUILD_TYPE}' mode as CMAKE_BUILD_TYPE was not specified." + ) + set(CMAKE_BUILD_TYPE + "${_DEFAULT_CMAKE_BUILD_TYPE}" + CACHE STRING "Choose the type of build." FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" + "MinSizeRel" "RelWithDebInfo") else() message(STATUS "Building SUNDIALS in '${CMAKE_BUILD_TYPE}' mode.") endif() @@ -186,14 +187,14 @@ include(SundialsSetupCompilers) include(SundialsSetupTPLs) # =============================================================== -# Build options to be proccessed last +# Build options to be processed last # =============================================================== include(SundialsBuildOptionsPost) # =============================================================== -# At this point all the configuration options are set. -# Setup the sundials_config.h. +# At this point all the configuration options are set. Setup the +# sundials_config.h. # =============================================================== include(SundialsSetupConfig) @@ -231,68 +232,56 @@ endif() # install sundials_export header file install(FILES "${PROJECT_BINARY_DIR}/include/sundials/sundials_export.h" - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/sundials" -) + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/sundials") # install configured header file install(FILES "${PROJECT_BINARY_DIR}/include/sundials/sundials_config.h" - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/sundials" -) + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/sundials") # install shared Fortran 2003 modules if(BUILD_FORTRAN_MODULE_INTERFACE) - # While the .mod files get generated for static and shared - # libraries, they are identical. So only install one set - # of the .mod files. + # While the .mod files get generated for static and shared libraries, they are + # identical. So only install one set of the .mod files. if(BUILD_STATIC_LIBS) install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}_STATIC/ - DESTINATION ${Fortran_INSTALL_MODDIR} - ) + DESTINATION ${Fortran_INSTALL_MODDIR}) else() install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}_SHARED/ - DESTINATION ${Fortran_INSTALL_MODDIR} - ) + DESTINATION ${Fortran_INSTALL_MODDIR}) endif() endif() # install license and notice files install(FILES "${PROJECT_SOURCE_DIR}/LICENSE" - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/sundials" -) + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/sundials") install(FILES "${PROJECT_SOURCE_DIR}/NOTICE" - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/sundials" -) + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/sundials") # create package version file include(CMakePackageConfigHelpers) write_basic_package_version_file( SUNDIALSConfigVersion.cmake VERSION ${PACKAGE_VERSION} - COMPATIBILITY AnyNewerVersion -) + COMPATIBILITY AnyNewerVersion) # install targets install( EXPORT sundials-targets FILE SUNDIALSTargets.cmake NAMESPACE SUNDIALS:: - DESTINATION "${SUNDIALS_INSTALL_CMAKEDIR}" -) + DESTINATION "${SUNDIALS_INSTALL_CMAKEDIR}") # install SUNDIALSConfig.cmake configure_package_config_file( "${PROJECT_SOURCE_DIR}/cmake/SUNDIALSConfig.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/SUNDIALSConfig.cmake" - INSTALL_DESTINATION "${SUNDIALS_INSTALL_CMAKEDIR}" -) + INSTALL_DESTINATION "${SUNDIALS_INSTALL_CMAKEDIR}") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/SUNDIALSConfig.cmake" "${CMAKE_CURRENT_BINARY_DIR}/SUNDIALSConfigVersion.cmake" - DESTINATION "${SUNDIALS_INSTALL_CMAKEDIR}" -) + DESTINATION "${SUNDIALS_INSTALL_CMAKEDIR}") # Export targets so build directory can be used directly export( EXPORT sundials-targets FILE "${CMAKE_CURRENT_BINARY_DIR}/SUNDIALSTargets.cmake" - NAMESPACE SUNDIALS:: -) + NAMESPACE SUNDIALS::) diff --git a/CODEOWNERS b/CODEOWNERS index 8d465ce4e5..e017386f2b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -3,6 +3,6 @@ # These owners will be the default owners for everything in # the repo. Unless a later match takes precedence, -# globabl owners will be requested for review when someone +# global owners will be requested for review when someone # opens a pull request. * @balos1 @gardner48 @cswoodward @drreynolds diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4106e3738b..be273c77a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to SUNDIALS -There are two primary ways of contributing to SUNDIALS. The first way is by particpating +There are two primary ways of contributing to SUNDIALS. The first way is by participating in the development of SUNDIALS directly through contributions of code to the primary [SUNDIALS repository](https://github.com/LLNL/sundials). This is the best way to contribute bug fixes and minor improvements. At this time, the SUNDIALS team does not have the resources diff --git a/README.md b/README.md index c410afb6ca..d00f7f276b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # SUNDIALS: SUite of Nonlinear and DIfferential/ALgebraic equation Solvers # -### Version 7.1.1 (Jun 2024) ### +### Version 7.2.0 (Dec 2024) ### **Center for Applied Scientific Computing, Lawrence Livermore National Laboratory** @@ -15,50 +15,62 @@ equation (ODE) systems, differential-algebraic equation (DAE) systems, and nonlinear algebraic systems: * ARKODE - for integrating stiff, nonstiff, and multirate ODEs of the form + $$M(t) \\, y' = f_1(t,y) + f_2(t,y), \quad y(t_0) = y_0$$ * CVODE - for integrating stiff and nonstiff ODEs of the form + $$y' = f(t,y), \quad y(t_0) = y_0$$ * CVODES - for integrating and sensitivity analysis (forward and adjoint) of ODEs of the form + $$y' = f(t,y,p), \quad y(t_0) = y_0(p)$$ * IDA - for integrating DAEs of the form + $$F(t,y,y') = 0, \quad y(t_0) = y_0, \quad y'(t_0) = y_0'$$ * IDAS - for integrating and sensitivity analysis (forward and adjoint) of DAEs of the form + $$F(t,y,y',p) = 0, \quad y(t_0) = y_0(p), \quad y'(t_0) = y_0'(p)$$ * KINSOL - for solving nonlinear algebraic systems of the form + $$F(u) = 0 \quad \text{or} \quad G(u) = u$$ ## Installation ## -For installation directions see the [online install guide](https://sundials.readthedocs.io/en/latest/Install_link.html), -the installation chapter in any of the package user guides, or INSTALL_GUIDE.pdf. +For installation directions, see the [getting started](https://sundials.readthedocs.io/en/latest/sundials/index.html#getting-started) +section in the online documentation. In the [released tarballs](https://github.com/LLNL/sundials/releases), +installation directions are also available in `INSTALL_GUIDE.pdf` and the +installation chapter of the user guides in the `doc` directory. Warning to users who receive more than one of the individual packages at -different times: Mixing old and new versions of SUNDIALS may fail. To avoid -such failures, obtain all desired package at the same time. +different times: Mixing old and new versions of SUNDIALS may fail. To avoid such +failures, obtain all desired package at the same time. ## Support ## -Full user guides for all of the SUNDIALS packages are available [online](https://sundials.readthedocs.io) -and in the [doc](./doc) directory. Additionally, the [doc](./doc) directory -contains documentation for the package example programs. +Full user guides for all of the SUNDIALS packages are available [online](https://sundials.readthedocs.io). +In the [released tarballs](https://github.com/LLNL/sundials/releases), the `doc` +directory includes PDFs of the user guides and documentation for the example +programs. The example program documentation PDFs are also available on the +[releases page](https://github.com/LLNL/sundials/releases). For information on recent changes to SUNDIALS see the [CHANGELOG](./CHANGELOG.md) or the introduction chapter of any package user guide. A list of Frequently Asked Questions on build and installation procedures as -well as common usage issues is available on the SUNDIALS [FAQ](https://computing.llnl.gov/projects/sundials/faq). -For dealing with systems with unphysical solutions or discontinuities see the -SUNDIALS [usage notes](https://computing.llnl.gov/projects/sundials/usage-notes). +well as common usage issues is available on the SUNDIALS +[FAQ](https://computing.llnl.gov/projects/sundials/faq). For dealing with +systems with nonphysical solutions or discontinuities see the SUNDIALS +[usage notes](https://computing.llnl.gov/projects/sundials/usage-notes). -If you have a question not covered in the FAQ or usage notes, please submit -your question to the SUNDIALS [mailing list](https://computing.llnl.gov/projects/sundials/mailing-list). +If you have a question not covered in the FAQ or usage notes, please submit your +question as a [GitHub issue](https://github.com/LLNL/sundials/issues) or to the +SUNDIALS [mailing list](https://computing.llnl.gov/projects/sundials/mailing-list). ## Contributing ## @@ -76,16 +88,33 @@ any publications reporting work done using SUNDIALS packages. The SUNDIALS library has been developed over many years by a number of contributors. The current SUNDIALS team consists of Cody J. Balos, -David J. Gardner, Alan C. Hindmarsh, Daniel R. Reynolds, and Carol S. Woodward. -We thank Radu Serban for significant and critical past contributions. +David J. Gardner, Alan C. Hindmarsh, Daniel R. Reynolds, Steven B. Roberts, and +Carol S. Woodward. We thank Radu Serban for significant and critical past +contributions. Other contributors to SUNDIALS include: James Almgren-Bell, Lawrence E. Banks, Peter N. Brown, George Byrne, Rujeko Chinomona, Scott D. Cohen, Aaron Collier, Keith E. Grant, Steven L. Lee, Shelby L. Lockhart, John Loffeld, Daniel McGreer, -Yu Pan, Slaven Peles, Cosmin Petra, Steven B. Roberts, H. Hunter Schwartz, -Jean M. Sexton, Dan Shumaker, Steve G. Smith, Shahbaj Sohal, Allan G. Taylor, +Yu Pan, Slaven Peles, Cosmin Petra, H. Hunter Schwartz, Jean M. Sexton, +Dan Shumaker, Steve G. Smith, Shahbaj Sohal, Allan G. Taylor, Hilari C. Tiedeman, Chris White, Ting Yan, and Ulrike M. Yang. +## Acknowledgements ## + +This material is based on work supported by the U.S. Department of Energy, +Office of Science, Office of Advanced Scientific Computing Research, Scientific +Discovery through Advanced Computing (SciDAC) program via the Frameworks, +Algorithms, and Scalable Technologies for Mathematics (FASTMath) Institute under +DOE awards DE-AC52-07NA27344 and DE-SC-0021354. + +This material is also based on work supported by the U.S. Department of Energy, +Office of Science, Office of Advanced Scientific Computing Research, +Next-Generation Scientific Software Technologies program under contract +DE-AC52-07NA27344. Additional support is also provided by SciDAC +partnerships with the U.S. Department of Energy’s FES, NP, BES, OE, and BER +offices as well as the LLNL Institutional Scientific Capability Portfolio. + + ## License ## SUNDIALS is released under the BSD 3-clause license. See the [LICENSE](./LICENSE) @@ -93,7 +122,7 @@ and [NOTICE](./NOTICE) files for details. All new contributions must be made under the BSD 3-clause license. **Please Note** If you are using SUNDIALS with any third party libraries linked -in (e.g., LAPACK, KLU, SuperLU_MT, PETSc, or *hypre*), be sure to review the +in (e.g., LAPACK, KLU, SuperLU_MT, PETSc, *hypre*, etc.), be sure to review the respective license of the package as that license may have more restrictive terms than the SUNDIALS license. diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 980e1e230d..2c139fc356 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -14,27 +14,29 @@ # benchmarks level CMakeLists.txt for SUNDIALS # --------------------------------------------------------------- -if(NOT (CMAKE_BUILD_TYPE STREQUAL "Release" OR - CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")) +if(NOT (CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL + "RelWithDebInfo")) message(WARNING "SUNDIALS is not being built in a \"Release\" mode, " - "benchmark performance will be affected") + "benchmark performance will be affected") endif() sundials_option(BENCHMARK_NVECTOR BOOL "NVector benchmarks are on" ON) # Disable some warnings for benchmarks if(ENABLE_ALL_WARNINGS) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-parameter") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter") + set(CMAKE_C_FLAGS + "${CMAKE_C_FLAGS} -Wno-unused-parameter -Wno-unused-function") + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-unused-function") endif() -#---------------------------------------- +# ---------------------------------------- # Add specific benchmarks -#---------------------------------------- +# ---------------------------------------- if(ENABLE_MPI) -add_subdirectory(diffusion_2D) -add_subdirectory(advection_reaction_3D) + add_subdirectory(diffusion_2D) + add_subdirectory(advection_reaction_3D) endif() # Add the nvector benchmarks diff --git a/benchmarks/advection_reaction_3D/CMakeLists.txt b/benchmarks/advection_reaction_3D/CMakeLists.txt index 79c616818a..b4b953a437 100644 --- a/benchmarks/advection_reaction_3D/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/CMakeLists.txt @@ -19,4 +19,3 @@ endif() if(ENABLE_KOKKOS AND BUILD_NVECTOR_KOKKOS) add_subdirectory(kokkos) endif() - diff --git a/benchmarks/advection_reaction_3D/kokkos/CMakeLists.txt b/benchmarks/advection_reaction_3D/kokkos/CMakeLists.txt index 8a12ba022b..1bb7ac2c5e 100644 --- a/benchmarks/advection_reaction_3D/kokkos/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/kokkos/CMakeLists.txt @@ -13,14 +13,17 @@ # --------------------------------------------------------------- # Add the build targets for each backend -if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) +if(BUILD_ARKODE + AND BUILD_CVODE + AND BUILD_IDA) foreach(backend ${KOKKOS_EXAMPLES_BACKENDS}) # set benchmark target name set(benchmark_target "advection_reaction_3D_kokkos.${backend}") # benchmark source files - add_executable(${benchmark_target} + add_executable( + ${benchmark_target} advection_reaction_3D.cpp arkode_driver.cpp cvode_driver.cpp @@ -29,32 +32,32 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) ParallelGrid.hpp check_retval.h) - # which backend to use - target_compile_definitions(${benchmark_target} PRIVATE USE_${backend}) - - # directories to include - target_include_directories(${benchmark_target} - PRIVATE - ${PROJECT_SOURCE_DIR}/utilities - ${MPI_CXX_INCLUDE_DIRS} - ) - - # libraries to link against - target_link_libraries(${benchmark_target} - PRIVATE - sundials_arkode - sundials_cvode - sundials_ida - sundials_nvecmpiplusx - sundials_nveckokkos - ${MPI_CXX_LIBRARIES} - ${EXE_EXTRA_LINK_LIBS} - ) - - install(TARGETS ${benchmark_target} + # which backend to use + target_compile_definitions(${benchmark_target} PRIVATE USE_${backend}) + + # directories to include + target_include_directories( + ${benchmark_target} PRIVATE ${PROJECT_SOURCE_DIR}/utilities + ${MPI_CXX_INCLUDE_DIRS}) + + # libraries to link against + target_link_libraries( + ${benchmark_target} + PRIVATE sundials_arkode + sundials_cvode + sundials_ida + sundials_nvecmpiplusx + sundials_nveckokkos + ${MPI_CXX_LIBRARIES} + ${EXE_EXTRA_LINK_LIBS}) + + install( + TARGETS ${benchmark_target} DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/kokkos") - install(FILES README.md ../scripts/compare_error.py ../scripts/compute_error.py ../scripts/pickle_solution_output.py + install( + FILES README.md ../scripts/compare_error.py ../scripts/compute_error.py + ../scripts/pickle_solution_output.py DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/kokkos") endforeach() diff --git a/benchmarks/advection_reaction_3D/kokkos/ParallelGrid.hpp b/benchmarks/advection_reaction_3D/kokkos/ParallelGrid.hpp index 7bbf8743a7..78fafd71ad 100644 --- a/benchmarks/advection_reaction_3D/kokkos/ParallelGrid.hpp +++ b/benchmarks/advection_reaction_3D/kokkos/ParallelGrid.hpp @@ -72,7 +72,7 @@ class ParallelGrid public: // Constructor that creates a new ParallelGrid object. // [in] - the memory helper to use for allocating the MPI buffers - // [in,out] comm - on input, the overal MPI communicator, on output, the cartesian communicator + // [in,out] comm - on input, the overall MPI communicator, on output, the cartesian communicator // [in] a[] - an array of length 3 which defines the domain [a,b] // [in] b[] - an array of length 3 which defines the domain [a,b] // [in] npts[] - an array of length 3 which defines the number of mesh points in each dimension @@ -563,7 +563,7 @@ class ParallelGrid GLOBALINT nx, ny, nz; /* number of intervals globally */ int nxl, nyl, nzl; /* number of intervals locally */ - int npx, npy, npz; /* numner of processes */ + int npx, npy, npz; /* number of processes */ sunrealtype dx, dy, dz; /* mesh spacing */ sunrealtype ax, ay, az; /* domain in [a, b] */ sunrealtype bx, by, bz; diff --git a/benchmarks/advection_reaction_3D/kokkos/arkode_driver.cpp b/benchmarks/advection_reaction_3D/kokkos/arkode_driver.cpp index 2001c2736a..8330eafb70 100644 --- a/benchmarks/advection_reaction_3D/kokkos/arkode_driver.cpp +++ b/benchmarks/advection_reaction_3D/kokkos/arkode_driver.cpp @@ -33,7 +33,7 @@ typedef struct SUNNonlinearSolver local_nls; }* TaskLocalNewton_Content; -/* Content accessor macors */ +/* Content accessor macros */ #define GET_NLS_CONTENT(NLS) ((TaskLocalNewton_Content)(NLS->content)) #define LOCAL_NLS(NLS) (GET_NLS_CONTENT(NLS)->local_nls) @@ -189,8 +189,10 @@ int EvolveProblemDIRK(N_Vector y, UserData* udata, UserOptions* uopt) check_retval(&retval, "ARKodeGetNumSteps", 1, udata->myid); retval = ARKodeGetNumStepAttempts(arkode_mem, &nst_a); check_retval(&retval, "ARKodeGetNumStepAttempts", 1, udata->myid); - retval = ARKStepGetNumRhsEvals(arkode_mem, &nfe, &nfi); - check_retval(&retval, "ARKStepGetNumRhsEvals", 1, udata->myid); + retval = ARKodeGetNumRhsEvals(arkode_mem, 0, &nfe); + check_retval(&retval, "ARKodeGetNumRhsEvals", 1, udata->myid); + retval = ARKodeGetNumRhsEvals(arkode_mem, 1, &nfi); + check_retval(&retval, "ARKodeGetNumRhsEvals", 1, udata->myid); retval = ARKodeGetNumErrTestFails(arkode_mem, &netf); check_retval(&retval, "ARKodeGetNumErrTestFails", 1, udata->myid); retval = ARKodeGetNumNonlinSolvIters(arkode_mem, &nni); @@ -392,8 +394,10 @@ int EvolveProblemIMEX(N_Vector y, UserData* udata, UserOptions* uopt) check_retval(&retval, "ARKodeGetNumSteps", 1, udata->myid); retval = ARKodeGetNumStepAttempts(arkode_mem, &nst_a); check_retval(&retval, "ARKodeGetNumStepAttempts", 1, udata->myid); - retval = ARKStepGetNumRhsEvals(arkode_mem, &nfe, &nfi); - check_retval(&retval, "ARKStepGetNumRhsEvals", 1, udata->myid); + retval = ARKodeGetNumRhsEvals(arkode_mem, 0, &nfe); + check_retval(&retval, "ARKodeGetNumRhsEvals", 1, udata->myid); + retval = ARKodeGetNumRhsEvals(arkode_mem, 1, &nfi); + check_retval(&retval, "ARKodeGetNumRhsEvals", 1, udata->myid); retval = ARKodeGetNumErrTestFails(arkode_mem, &netf); check_retval(&retval, "ARKodeGetNumErrTestFails", 1, udata->myid); retval = ARKodeGetNumNonlinSolvIters(arkode_mem, &nni); @@ -516,8 +520,8 @@ int EvolveProblemExplicit(N_Vector y, UserData* udata, UserOptions* uopt) check_retval(&retval, "ARKodeGetNumSteps", 1, udata->myid); retval = ARKodeGetNumStepAttempts(arkode_mem, &nst_a); check_retval(&retval, "ARKodeGetNumStepAttempts", 1, udata->myid); - retval = ERKStepGetNumRhsEvals(arkode_mem, &nfe); - check_retval(&retval, "ERKStepGetNumRhsEvals", 1, udata->myid); + retval = ARKodeGetNumRhsEvals(arkode_mem, 0, &nfe); + check_retval(&retval, "ARKodeGetNumRhsEvals", 1, udata->myid); retval = ARKodeGetNumErrTestFails(arkode_mem, &netf); check_retval(&retval, "ARKodeGetNumErrTestFails", 1, udata->myid); diff --git a/benchmarks/advection_reaction_3D/kokkos/check_retval.h b/benchmarks/advection_reaction_3D/kokkos/check_retval.h index b72575c517..a64f816976 100644 --- a/benchmarks/advection_reaction_3D/kokkos/check_retval.h +++ b/benchmarks/advection_reaction_3D/kokkos/check_retval.h @@ -23,7 +23,7 @@ * opt == 0 means the function allocates memory and returns a * pointer so check if a NULL pointer was returned * opt == 1 means the function returns an integer where a - * value < 0 indicates an error occured + * value < 0 indicates an error occurred * --------------------------------------------------------------*/ static int check_retval(void* returnvalue, const char* funcname, int opt, int myid) { diff --git a/benchmarks/advection_reaction_3D/kokkos/rhs3D.hpp b/benchmarks/advection_reaction_3D/kokkos/rhs3D.hpp index 0dee917835..ea714ecaeb 100644 --- a/benchmarks/advection_reaction_3D/kokkos/rhs3D.hpp +++ b/benchmarks/advection_reaction_3D/kokkos/rhs3D.hpp @@ -23,7 +23,7 @@ * --------------------------------------------------------------*/ /* Compute the advection term f(t,y) = -c (grad * y). This is done using - upwind 1st order finite differences. At present, only periodic boudary + upwind 1st order finite differences. At present, only periodic boundary conditions are supported, which are handled via MPI's Cartesian communicator (even for serial runs). */ static int Advection(sunrealtype t, N_Vector y, N_Vector ydot, void* user_data) diff --git a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt index d816b35124..264e023b5a 100644 --- a/benchmarks/advection_reaction_3D/raja/CMakeLists.txt +++ b/benchmarks/advection_reaction_3D/raja/CMakeLists.txt @@ -13,24 +13,28 @@ # SUNDIALS Copyright End # --------------------------------------------------------------- -if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) +if(BUILD_ARKODE + AND BUILD_CVODE + AND BUILD_IDA) - if((RAJA_BACKENDS MATCHES "TARGET_OPENMP") OR (RAJA_BACKENDS MATCHES "OPENMP")) + if((RAJA_BACKENDS MATCHES "TARGET_OPENMP") OR (RAJA_BACKENDS MATCHES "OPENMP" + )) set(OTHER_LIBS OpenMP::OpenMP_CXX) endif() # Set up parameters to run benchmarks with set(BENCHMARK_VAR - "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton" - "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton" - "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton" - "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton") + "--method ARK-IMEX --nls tl-newton --tf 0.01 --dont-save\;arkimex_tlnewton" + "--method ARK-DIRK --nls newton --tf 0.01 --dont-save\;arkdirk_newton" + "--method CV-BDF --nls newton --tf 0.01 --dont-save\;cvbdf_newton" + "--method IDA --nls newton --tf 0.01 --dont-save\;ida_newton") # ---------------------------------------------------------------------------- # MPI only # ---------------------------------------------------------------------------- - add_executable(advection_reaction_3D_raja + add_executable( + advection_reaction_3D_raja advection_reaction_3D.cpp arkode_driver.cpp cvode_driver.cpp @@ -41,39 +45,41 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) backends.hpp) # ensure the linker language is reset to CXX - set_target_properties(advection_reaction_3D_raja PROPERTIES LINKER_LANGUAGE CXX) - - target_include_directories(advection_reaction_3D_raja - PRIVATE - ${PROJECT_SOURCE_DIR}/utilities - ${MPI_CXX_INCLUDE_DIRS}) - - target_link_libraries(advection_reaction_3D_raja - PRIVATE - sundials_arkode - sundials_cvode - sundials_ida - sundials_nvecmpiplusx - sundials_nvecserial - RAJA - ${MPI_CXX_LIBRARIES} - ${OTHER_LIBS}) + set_target_properties(advection_reaction_3D_raja PROPERTIES LINKER_LANGUAGE + CXX) + + target_include_directories( + advection_reaction_3D_raja PRIVATE ${PROJECT_SOURCE_DIR}/utilities + ${MPI_CXX_INCLUDE_DIRS}) + + target_link_libraries( + advection_reaction_3D_raja + PRIVATE sundials_arkode + sundials_cvode + sundials_ida + sundials_nvecmpiplusx + sundials_nvecserial + RAJA + ${MPI_CXX_LIBRARIES} + ${OTHER_LIBS}) install(TARGETS advection_reaction_3D_raja - DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") + DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") - install(FILES README.md ../scripts/compare_error.py ../scripts/compute_error.py ../scripts/pickle_solution_output.py - DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") + install(FILES README.md ../scripts/compare_error.py + ../scripts/compute_error.py ../scripts/pickle_solution_output.py + DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) - sundials_add_benchmark(advection_reaction_3D_raja advection_reaction_3D_raja advection_reaction_3D + sundials_add_benchmark( + advection_reaction_3D_raja advection_reaction_3D_raja + advection_reaction_3D NUM_CORES ${SUNDIALS_BENCHMARK_NUM_CPUS} BENCHMARK_ARGS ${benchmark_args} - IDENTIFIER ${identifier} - ) + IDENTIFIER ${identifier}) endforeach() # ---------------------------------------------------------------------------- @@ -82,13 +88,14 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) if(BUILD_NVECTOR_CUDA) - set_source_files_properties(advection_reaction_3D.cpp - PROPERTIES LANGUAGE CUDA) + set_source_files_properties(advection_reaction_3D.cpp PROPERTIES LANGUAGE + CUDA) set_source_files_properties(arkode_driver.cpp PROPERTIES LANGUAGE CUDA) set_source_files_properties(cvode_driver.cpp PROPERTIES LANGUAGE CUDA) set_source_files_properties(ida_driver.cpp PROPERTIES LANGUAGE CUDA) - add_executable(advection_reaction_3D_raja_mpicuda + add_executable( + advection_reaction_3D_raja_mpicuda advection_reaction_3D.cpp arkode_driver.cpp cvode_driver.cpp @@ -100,35 +107,36 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) # ensure the linker language is reset to CXX set_target_properties(advection_reaction_3D_raja_mpicuda - PROPERTIES LINKER_LANGUAGE CXX) - - target_include_directories(advection_reaction_3D_raja_mpicuda - PRIVATE - ${PROJECT_SOURCE_DIR}/utilities - ${MPI_CXX_INCLUDE_DIRS}) - - target_link_libraries(advection_reaction_3D_raja_mpicuda - PRIVATE - sundials_arkode - sundials_cvode - sundials_ida - sundials_nvecmpiplusx - sundials_nveccuda - RAJA - ${MPI_CXX_LIBRARIES} - ${OTHER_LIBS}) - - target_compile_definitions(advection_reaction_3D_raja_mpicuda PRIVATE USE_CUDA_NVEC) + PROPERTIES LINKER_LANGUAGE CXX) + + target_include_directories( + advection_reaction_3D_raja_mpicuda PRIVATE ${PROJECT_SOURCE_DIR}/utilities + ${MPI_CXX_INCLUDE_DIRS}) + + target_link_libraries( + advection_reaction_3D_raja_mpicuda + PRIVATE sundials_arkode + sundials_cvode + sundials_ida + sundials_nvecmpiplusx + sundials_nveccuda + RAJA + ${MPI_CXX_LIBRARIES} + ${OTHER_LIBS}) + + target_compile_definitions(advection_reaction_3D_raja_mpicuda + PRIVATE USE_CUDA_NVEC) install(TARGETS advection_reaction_3D_raja_mpicuda - DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") + DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) - sundials_add_benchmark(advection_reaction_3D_raja_mpicuda advection_reaction_3D_raja_mpicuda advection_reaction_3D - ENABLE_GPU + sundials_add_benchmark( + advection_reaction_3D_raja_mpicuda advection_reaction_3D_raja_mpicuda + advection_reaction_3D ENABLE_GPU NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} BENCHMARK_ARGS ${benchmark_args} IDENTIFIER ${identifier}) @@ -141,7 +149,8 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) if(BUILD_NVECTOR_HIP) - add_executable(advection_reaction_3D_raja_mpihip + add_executable( + advection_reaction_3D_raja_mpihip advection_reaction_3D.cpp advection_reaction_3D.hpp arkode_driver.cpp @@ -152,34 +161,35 @@ if(BUILD_ARKODE AND BUILD_CVODE AND BUILD_IDA) check_retval.h backends.hpp) - target_include_directories(advection_reaction_3D_raja_mpihip - PRIVATE - ${PROJECT_SOURCE_DIR}/utilities - ${MPI_CXX_INCLUDE_DIRS}) - - target_link_libraries(advection_reaction_3D_raja_mpihip - PRIVATE - sundials_arkode - sundials_cvode - sundials_ida - sundials_nvecmpiplusx - sundials_nvechip - RAJA - hip::device - ${MPI_CXX_LIBRARIES} - ${OTHER_LIBS}) - - target_compile_definitions(advection_reaction_3D_raja_mpihip PRIVATE USE_HIP_NVEC) + target_include_directories( + advection_reaction_3D_raja_mpihip PRIVATE ${PROJECT_SOURCE_DIR}/utilities + ${MPI_CXX_INCLUDE_DIRS}) + + target_link_libraries( + advection_reaction_3D_raja_mpihip + PRIVATE sundials_arkode + sundials_cvode + sundials_ida + sundials_nvecmpiplusx + sundials_nvechip + RAJA + hip::device + ${MPI_CXX_LIBRARIES} + ${OTHER_LIBS}) + + target_compile_definitions(advection_reaction_3D_raja_mpihip + PRIVATE USE_HIP_NVEC) install(TARGETS advection_reaction_3D_raja_mpihip - DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") + DESTINATION "${BENCHMARKS_INSTALL_PATH}/advection_reaction_3D/raja") foreach(benchmark_tuple ${BENCHMARK_VAR}) list(GET benchmark_tuple 0 benchmark_args) list(GET benchmark_tuple 1 identifier) - sundials_add_benchmark(advection_reaction_3D_raja_mpihip advection_reaction_3D_raja_mpihip advection_reaction_3D - ENABLE_GPU + sundials_add_benchmark( + advection_reaction_3D_raja_mpihip advection_reaction_3D_raja_mpihip + advection_reaction_3D ENABLE_GPU NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} BENCHMARK_ARGS ${benchmark_args} IDENTIFIER ${identifier}) diff --git a/benchmarks/advection_reaction_3D/raja/ParallelGrid.hpp b/benchmarks/advection_reaction_3D/raja/ParallelGrid.hpp index e818194047..f234aa089a 100644 --- a/benchmarks/advection_reaction_3D/raja/ParallelGrid.hpp +++ b/benchmarks/advection_reaction_3D/raja/ParallelGrid.hpp @@ -45,7 +45,7 @@ class ParallelGrid public: // Constructor that creates a new ParallelGrid object. // [in] - the memory helper to use for allocating the MPI buffers - // [in,out] comm - on input, the overal MPI communicator, on output, the cartesian communicator + // [in,out] comm - on input, the overall MPI communicator, on output, the cartesian communicator // [in] a[] - an array of length 3 which defines the domain [a,b] // [in] b[] - an array of length 3 which defines the domain [a,b] // [in] npts[] - an array of length 3 which defines the number of mesh points in each dimension @@ -508,7 +508,7 @@ class ParallelGrid GLOBALINT nx, ny, nz; /* number of intervals globally */ int nxl, nyl, nzl; /* number of intervals locally */ - int npx, npy, npz; /* numner of processes */ + int npx, npy, npz; /* number of processes */ REAL dx, dy, dz; /* mesh spacing */ REAL ax, ay, az; /* domain in [a, b] */ REAL bx, by, bz; diff --git a/benchmarks/advection_reaction_3D/raja/advection_reaction_3D.cpp b/benchmarks/advection_reaction_3D/raja/advection_reaction_3D.cpp index 7ee19de56d..770f1f3a54 100644 --- a/benchmarks/advection_reaction_3D/raja/advection_reaction_3D.cpp +++ b/benchmarks/advection_reaction_3D/raja/advection_reaction_3D.cpp @@ -483,7 +483,7 @@ int SetupProblem(int argc, char* argv[], UserData* udata, UserOptions* uopt, uopt->fused = 0; /* use fused vector ops */ uopt->save = 1; /* save solution to disk */ uopt->nout = 10; /* number of output times */ - uopt->outputdir = (char*)"."; /* output directory */ + uopt->outputdir = "."; /* output directory */ /* Parse CLI args and set udata/uopt appropriately */ int retval = ParseArgs(argc, argv, udata, uopt); diff --git a/benchmarks/advection_reaction_3D/raja/advection_reaction_3D.hpp b/benchmarks/advection_reaction_3D/raja/advection_reaction_3D.hpp index 8499be6f0b..791686d9ca 100644 --- a/benchmarks/advection_reaction_3D/raja/advection_reaction_3D.hpp +++ b/benchmarks/advection_reaction_3D/raja/advection_reaction_3D.hpp @@ -57,7 +57,7 @@ struct UserOptions int fused; /* use fused vector ops */ int nout; /* number of outputs */ int save; /* save solution to disk */ - char* outputdir; + const char* outputdir; }; /* @@ -89,7 +89,7 @@ struct UserData N_Vector vmask; N_Vector wmask; - /* problem paramaters */ + /* problem parameters */ sunrealtype xmax; /* maximum x value */ sunrealtype A; /* concentration of species A */ sunrealtype B; /* w source rate */ diff --git a/benchmarks/advection_reaction_3D/raja/arkode_driver.cpp b/benchmarks/advection_reaction_3D/raja/arkode_driver.cpp index 49dcc6d8cf..2a77a3b89b 100644 --- a/benchmarks/advection_reaction_3D/raja/arkode_driver.cpp +++ b/benchmarks/advection_reaction_3D/raja/arkode_driver.cpp @@ -33,7 +33,7 @@ typedef struct SUNNonlinearSolver local_nls; }* TaskLocalNewton_Content; -/* Content accessor macors */ +/* Content accessor macros */ #define GET_NLS_CONTENT(NLS) ((TaskLocalNewton_Content)(NLS->content)) #define LOCAL_NLS(NLS) (GET_NLS_CONTENT(NLS)->local_nls) @@ -58,7 +58,6 @@ int EvolveProblemDIRK(N_Vector y, UserData* udata, UserOptions* uopt) long int nfe, nfi; /* RHS stats */ long int nni, ncnf; /* nonlinear solver stats */ long int nli, npsol; /* linear solver stats */ - char fname[MXSTR]; /* Additively split methods should not add the advection and reaction terms */ udata->add_reactions = true; @@ -190,8 +189,10 @@ int EvolveProblemDIRK(N_Vector y, UserData* udata, UserOptions* uopt) check_retval(&retval, "ARKodeGetNumSteps", 1, udata->myid); retval = ARKodeGetNumStepAttempts(arkode_mem, &nst_a); check_retval(&retval, "ARKodeGetNumStepAttempts", 1, udata->myid); - retval = ARKStepGetNumRhsEvals(arkode_mem, &nfe, &nfi); - check_retval(&retval, "ARKStepGetNumRhsEvals", 1, udata->myid); + retval = ARKodeGetNumRhsEvals(arkode_mem, 0, &nfe); + check_retval(&retval, "ARKodeGetNumRhsEvals", 1, udata->myid); + retval = ARKodeGetNumRhsEvals(arkode_mem, 1, &nfi); + check_retval(&retval, "ARKodeGetNumRhsEvals", 1, udata->myid); retval = ARKodeGetNumErrTestFails(arkode_mem, &netf); check_retval(&retval, "ARKodeGetNumErrTestFails", 1, udata->myid); retval = ARKodeGetNumNonlinSolvIters(arkode_mem, &nni); @@ -246,7 +247,6 @@ int EvolveProblemIMEX(N_Vector y, UserData* udata, UserOptions* uopt) long int nfe, nfi; /* RHS stats */ long int nni, ncnf; /* nonlinear solver stats */ long int nli, npsol; /* linear solver stats */ - char fname[MXSTR]; /* Additively split methods should not add the advection and reaction terms */ udata->add_reactions = false; @@ -394,8 +394,10 @@ int EvolveProblemIMEX(N_Vector y, UserData* udata, UserOptions* uopt) check_retval(&retval, "ARKodeGetNumSteps", 1, udata->myid); retval = ARKodeGetNumStepAttempts(arkode_mem, &nst_a); check_retval(&retval, "ARKodeGetNumStepAttempts", 1, udata->myid); - retval = ARKStepGetNumRhsEvals(arkode_mem, &nfe, &nfi); - check_retval(&retval, "ARKStepGetNumRhsEvals", 1, udata->myid); + retval = ARKodeGetNumRhsEvals(arkode_mem, 0, &nfe); + check_retval(&retval, "ARKodeGetNumRhsEvals", 1, udata->myid); + retval = ARKodeGetNumRhsEvals(arkode_mem, 1, &nfi); + check_retval(&retval, "ARKodeGetNumRhsEvals", 1, udata->myid); retval = ARKodeGetNumErrTestFails(arkode_mem, &netf); check_retval(&retval, "ARKodeGetNumErrTestFails", 1, udata->myid); retval = ARKodeGetNumNonlinSolvIters(arkode_mem, &nni); @@ -445,7 +447,6 @@ int EvolveProblemExplicit(N_Vector y, UserData* udata, UserOptions* uopt) int iout; /* output counter */ long int nst, nst_a, netf; /* step stats */ long int nfe; /* RHS stats */ - char fname[MXSTR]; /* Additively split methods should not add the advection and reaction terms */ udata->add_reactions = true; @@ -519,8 +520,8 @@ int EvolveProblemExplicit(N_Vector y, UserData* udata, UserOptions* uopt) check_retval(&retval, "ARKodeGetNumSteps", 1, udata->myid); retval = ARKodeGetNumStepAttempts(arkode_mem, &nst_a); check_retval(&retval, "ARKodeGetNumStepAttempts", 1, udata->myid); - retval = ERKStepGetNumRhsEvals(arkode_mem, &nfe); - check_retval(&retval, "ERKStepGetNumRhsEvals", 1, udata->myid); + retval = ARKodeGetNumRhsEvals(arkode_mem, 0, &nfe); + check_retval(&retval, "ARKodeGetNumRhsEvals", 1, udata->myid); retval = ARKodeGetNumErrTestFails(arkode_mem, &netf); check_retval(&retval, "ARKodeGetNumErrTestFails", 1, udata->myid); diff --git a/benchmarks/advection_reaction_3D/raja/backends.hpp b/benchmarks/advection_reaction_3D/raja/backends.hpp index c7cbd790ee..c258c68598 100644 --- a/benchmarks/advection_reaction_3D/raja/backends.hpp +++ b/benchmarks/advection_reaction_3D/raja/backends.hpp @@ -114,12 +114,23 @@ using XYZ_KERNEL_POL = #else #define NVECTOR_ID_STRING "Serial" -using EXEC_POLICY = RAJA::seq_exec; -using XYZ_KERNEL_POL = RAJA::KernelPolicy>>>>; +#else +using EXEC_POLICY = RAJA::seq_exec; +using XYZ_KERNEL_POL = RAJA::KernelPolicy>>>>; +#endif + constexpr auto LocalNvector = N_VNew_Serial; #define CopyVecFromDevice(v) diff --git a/benchmarks/advection_reaction_3D/raja/check_retval.h b/benchmarks/advection_reaction_3D/raja/check_retval.h index b72575c517..a64f816976 100644 --- a/benchmarks/advection_reaction_3D/raja/check_retval.h +++ b/benchmarks/advection_reaction_3D/raja/check_retval.h @@ -23,7 +23,7 @@ * opt == 0 means the function allocates memory and returns a * pointer so check if a NULL pointer was returned * opt == 1 means the function returns an integer where a - * value < 0 indicates an error occured + * value < 0 indicates an error occurred * --------------------------------------------------------------*/ static int check_retval(void* returnvalue, const char* funcname, int opt, int myid) { diff --git a/benchmarks/advection_reaction_3D/scripts/compare_error.py b/benchmarks/advection_reaction_3D/scripts/compare_error.py index 2dc66d23fa..4dd1ff7ee1 100755 --- a/benchmarks/advection_reaction_3D/scripts/compare_error.py +++ b/benchmarks/advection_reaction_3D/scripts/compare_error.py @@ -15,7 +15,8 @@ import glob import sys import matplotlib -matplotlib.use('Agg') + +matplotlib.use("Agg") from mpl_toolkits.mplot3d import Axes3D import matplotlib.pyplot as plt import pandas as pd @@ -25,57 +26,57 @@ # load pickled data def load_data(file): data = np.load(file) - m = data['mesh'] - t = data['t'] - u = data['u'] - v = data['v'] - w = data['w'] + m = data["mesh"] + t = data["t"] + u = data["u"] + v = data["v"] + w = data["w"] - hx = m[0,1] - m[0,0] - hy = m[1,1] - m[1,0] - hz = m[2,1] - m[2,0] + hx = m[0, 1] - m[0, 0] + hy = m[1, 1] - m[1, 0] + hz = m[2, 1] - m[2, 0] - return { 'm': m, 'h': (hx,hy,hz), 't': t, 'u': u, 'v': v, 'w': w } + return {"m": m, "h": (hx, hy, hz), "t": t, "u": u, "v": v, "w": w} # grid function norm def norm_3Dgrid(h, x, q=1): - hx,hy,hz = h + hx, hy, hz = h s = np.shape(x) - return (hx*hy*hz*np.sum(np.abs(x)**q, axis=(1,2,3)))**(1./q) + return (hx * hy * hz * np.sum(np.abs(x) ** q, axis=(1, 2, 3))) ** (1.0 / q) # load data files -np111 = load_data('np-111/output-with-h-8.33e-02.npz') -np211 = load_data('np-211/output-with-h-8.33e-02.npz') -np311 = load_data('np-311/output-with-h-8.33e-02.npz') -np131 = load_data('np-131/output-with-h-8.33e-02.npz') -np113 = load_data('np-113/output-with-h-8.33e-02.npz') -np911 = load_data('np-911/output-with-h-8.33e-02.npz') +np111 = load_data("np-111/output-with-h-8.33e-02.npz") +np211 = load_data("np-211/output-with-h-8.33e-02.npz") +np311 = load_data("np-311/output-with-h-8.33e-02.npz") +np131 = load_data("np-131/output-with-h-8.33e-02.npz") +np113 = load_data("np-113/output-with-h-8.33e-02.npz") +np911 = load_data("np-911/output-with-h-8.33e-02.npz") # np133 = load_data('np-133/output-with-h-8.33e-02.npz') -np313 = load_data('np-313/output-with-h-8.33e-02.npz') -np331 = load_data('np-331/output-with-h-8.33e-02.npz') -np333 = load_data('np-333/output-with-h-8.33e-02.npz') +np313 = load_data("np-313/output-with-h-8.33e-02.npz") +np331 = load_data("np-331/output-with-h-8.33e-02.npz") +np333 = load_data("np-333/output-with-h-8.33e-02.npz") # np666 = load_data('np-666/output-with-h-8.33e-02.npz') -for component in ['u', 'v', 'w']: +for component in ["u", "v", "w"]: # Reference solution ref = np111[component] # Now compute E(h) = ||U(h) - \bar{U}(h)|| using the grid-function norm - E_np211 = norm_3Dgrid(np211['h'], np211[component] - ref) - E_np311 = norm_3Dgrid(np311['h'], np311[component] - ref) - E_np131 = norm_3Dgrid(np131['h'], np131[component] - ref) - E_np113 = norm_3Dgrid(np113['h'], np113[component] - ref) - E_np911 = norm_3Dgrid(np911['h'], np911[component] - ref) + E_np211 = norm_3Dgrid(np211["h"], np211[component] - ref) + E_np311 = norm_3Dgrid(np311["h"], np311[component] - ref) + E_np131 = norm_3Dgrid(np131["h"], np131[component] - ref) + E_np113 = norm_3Dgrid(np113["h"], np113[component] - ref) + E_np911 = norm_3Dgrid(np911["h"], np911[component] - ref) # E_np133 = norm_3Dgrid(np133['h'], np133[component] - ref) - E_np313 = norm_3Dgrid(np313['h'], np313[component] - ref) - E_np331 = norm_3Dgrid(np331['h'], np331[component] - ref) - E_np333 = norm_3Dgrid(np333['h'], np333[component] - ref) + E_np313 = norm_3Dgrid(np313["h"], np313[component] - ref) + E_np331 = norm_3Dgrid(np331["h"], np331[component] - ref) + E_np333 = norm_3Dgrid(np333["h"], np333[component] - ref) # E_np666 = norm_3Dgrid(np666['h'], np666[component] - ref) # Plot error across time - X, Y = np.meshgrid(np111['m'][0,:], np111['t']) + X, Y = np.meshgrid(np111["m"][0, :], np111["t"]) # fig = plt.figure() # ax = plt.subplot(311, projection='3d') # ax.plot_surface(X, Y, np.abs(np911[component][:,:,0,0] - ref[:,:,0,0])) @@ -83,17 +84,17 @@ def norm_3Dgrid(h, x, q=1): # ax.plot_surface(X, Y, np.abs(np911[component][:,0,:,0] - ref[:,0,:,0])) # ax = plt.subplot(313, projection='3d') # ax.plot_surface(X, Y, np.abs(np911[component][:,0,0,:] - ref[:,0,0,:])) - plt.plot(np111['t'], E_np211) - plt.plot(np111['t'], E_np131) - plt.plot(np111['t'], E_np113) - plt.plot(np111['t'], E_np911) + plt.plot(np111["t"], E_np211) + plt.plot(np111["t"], E_np131) + plt.plot(np111["t"], E_np113) + plt.plot(np111["t"], E_np911) # plt.plot(np111['t'], E_np133) - plt.plot(np111['t'], E_np313) - plt.plot(np111['t'], E_np331) - plt.plot(np111['t'], E_np333) + plt.plot(np111["t"], E_np313) + plt.plot(np111["t"], E_np331) + plt.plot(np111["t"], E_np333) # plt.plot(np111['t'], E_np666) # plt.legend(['2 1 1', '3 1 1', '1 3 3', '3 1 3', '3 3 1', '3 3 3', '6 6 6']) # plt.legend(['3 1 1', '1 3 1', '1 1 3', '9 1 1', '1 3 3', '3 1 3', '3 3 1']) - plt.ylabel('||E(hx,hy,hz)||') - plt.xlabel('time') - plt.savefig('compare-error-plot-%s.png' % component) + plt.ylabel("||E(hx,hy,hz)||") + plt.xlabel("time") + plt.savefig("compare-error-plot-%s.png" % component) diff --git a/benchmarks/advection_reaction_3D/scripts/compute_error.py b/benchmarks/advection_reaction_3D/scripts/compute_error.py index 2c01826b29..85f151ed59 100755 --- a/benchmarks/advection_reaction_3D/scripts/compute_error.py +++ b/benchmarks/advection_reaction_3D/scripts/compute_error.py @@ -15,7 +15,8 @@ import glob import sys import matplotlib -matplotlib.use('Agg') + +matplotlib.use("Agg") import matplotlib.pyplot as plt import pandas as pd import numpy as np @@ -24,65 +25,67 @@ # load pickled data def load_data(file): data = np.load(file) - m = data['mesh'] - t = data['t'] - u = data['u'] - v = data['v'] - w = data['w'] + m = data["mesh"] + t = data["t"] + u = data["u"] + v = data["v"] + w = data["w"] - hx = m[0,1] - m[0,0] - hy = m[1,1] - m[1,0] - hz = m[2,1] - m[2,0] + hx = m[0, 1] - m[0, 0] + hy = m[1, 1] - m[1, 0] + hz = m[2, 1] - m[2, 0] - return { 'm': m, 'h': (hx,hy,hz), 't': t, 'u': u, 'v': v, 'w': w } + return {"m": m, "h": (hx, hy, hz), "t": t, "u": u, "v": v, "w": w} # grid function norm def norm_3Dgrid(h, x, q=1): - hx,hy,hz = h - return (hx*hy*hz*np.sum(np.abs(x)**q, axis=(1,2,3)))**(1/q) + hx, hy, hz = h + return (hx * hy * hz * np.sum(np.abs(x) ** q, axis=(1, 2, 3))) ** (1 / q) # computer order of accuracy p def calc_order(h1, Eh1, h2, Eh2): - return np.log( Eh1/Eh2 ) / np.log( np.prod(h1)/np.prod(h2) ) + return np.log(Eh1 / Eh2) / np.log(np.prod(h1) / np.prod(h2)) # load data files -h_over_8 = load_data('middle-h/output-with-h-1.04e-02.npz') -h_over_4 = load_data('large-h/output-with-h-2.08e-02.npz') +h_over_8 = load_data("middle-h/output-with-h-1.04e-02.npz") +h_over_4 = load_data("large-h/output-with-h-2.08e-02.npz") # h_over_2 = load_data('larger-h/output-with-h-4.16e-02.npz') -h_over_1 = load_data('largest-h/output-with-h-8.33e-02.npz') +h_over_1 = load_data("largest-h/output-with-h-8.33e-02.npz") -for component in ['u', 'v', 'w']: +for component in ["u", "v", "w"]: # Restrict reference to the coarsest grid - ref = h_over_8[component][:,::8,::8,::8] + ref = h_over_8[component][:, ::8, ::8, ::8] # Now compute E(h) = ||U(h) - \bar{U}(h)|| using the grid-function norm - Eh_over_4 = norm_3Dgrid(h_over_4['h'], h_over_4[component][:,::4,::4,::4] - ref) - Eh_over_1 = norm_3Dgrid(h_over_1['h'], h_over_1[component][:,:,:,:] - ref) + Eh_over_4 = norm_3Dgrid(h_over_4["h"], h_over_4[component][:, ::4, ::4, ::4] - ref) + Eh_over_1 = norm_3Dgrid(h_over_1["h"], h_over_1[component][:, :, :, :] - ref) # Compute order p as in O(h^p) - p = calc_order(h_over_1['h'], Eh_over_1, h_over_4['h'], Eh_over_4) - print('min p for %s component: %.4f' % (component, np.min(p))) + p = calc_order(h_over_1["h"], Eh_over_1, h_over_4["h"], Eh_over_4) + print("min p for %s component: %.4f" % (component, np.min(p))) # Plot error across time plt.figure() - plt.plot(h_over_8['t'], Eh_over_4, 'r-') - plt.plot(h_over_8['t'], Eh_over_1, 'b-') - plt.ylabel('||E(hx,hy,hz)||') - plt.xlabel('time') - plt.savefig('error-in-time-plot-%s.png' % component) + plt.plot(h_over_8["t"], Eh_over_4, "r-") + plt.plot(h_over_8["t"], Eh_over_1, "b-") + plt.ylabel("||E(hx,hy,hz)||") + plt.xlabel("time") + plt.savefig("error-in-time-plot-%s.png" % component) # Plot error norm with respect to h plt.figure() - x = np.array([np.prod(h_over_4['h']), np.prod(h_over_1['h'])]) - plt.plot(x, x, 'k-') - plt.plot(x, x**2, 'k-') - plt.plot(x, [np.linalg.norm(Eh_over_4, np.Inf), np.linalg.norm(Eh_over_1, np.Inf)], 'r-') - plt.legend(['1st order', '2nd order', 'actual']) - plt.ylabel('|| ||E(hx,hy,hz)|| ||_inf') - plt.xlabel('hx * hy * hz') - plt.yscale('log') - plt.xscale('log') - plt.savefig('error-plot-%s.png' % component) + x = np.array([np.prod(h_over_4["h"]), np.prod(h_over_1["h"])]) + plt.plot(x, x, "k-") + plt.plot(x, x**2, "k-") + plt.plot( + x, [np.linalg.norm(Eh_over_4, np.Inf), np.linalg.norm(Eh_over_1, np.Inf)], "r-" + ) + plt.legend(["1st order", "2nd order", "actual"]) + plt.ylabel("|| ||E(hx,hy,hz)|| ||_inf") + plt.xlabel("hx * hy * hz") + plt.yscale("log") + plt.xscale("log") + plt.savefig("error-plot-%s.png" % component) diff --git a/benchmarks/advection_reaction_3D/scripts/make_plots.py b/benchmarks/advection_reaction_3D/scripts/make_plots.py index 69a0168d79..a4dfa87840 100755 --- a/benchmarks/advection_reaction_3D/scripts/make_plots.py +++ b/benchmarks/advection_reaction_3D/scripts/make_plots.py @@ -22,218 +22,265 @@ # ------------------------------------------------------------------------------ + # utility functions def parallel_coords(rank): - if (rank == 0): + if rank == 0: return [0, 0, 0] - if (rank == 1): + if rank == 1: return [0, 0, 1] - if (rank == 2): + if rank == 2: return [0, 1, 0] - if (rank == 3): + if rank == 3: return [0, 1, 1] - if (rank == 4): + if rank == 4: return [1, 0, 0] - if (rank == 5): + if rank == 5: return [1, 0, 1] - if (rank == 6): + if rank == 6: return [1, 1, 0] - if (rank == 7): + if rank == 7: return [1, 1, 1] -def xslice(u,it,ix): - return u[it,ix,:,:] -def yslice(u,it,iy): - return u[it,:,iy,:] +def xslice(u, it, ix): + return u[it, ix, :, :] + + +def yslice(u, it, iy): + return u[it, :, iy, :] + + +def zslice(u, it, iz): + return u[it, :, :, iz] -def zslice(u,it,iz): - return u[it,:,:,iz] -def xproj(u,it): - return np.average(u[it,:,:,:], axis=0) +def xproj(u, it): + return np.average(u[it, :, :, :], axis=0) -def yproj(u,it): - return np.average(u[it,:,:,:], axis=1) -def zproj(u,it): - return np.average(u[it,:,:,:], axis=2) +def yproj(u, it): + return np.average(u[it, :, :, :], axis=1) -def myplot(axis, X, Y, Z, xlabel='none', ylabel='none'): + +def zproj(u, it): + return np.average(u[it, :, :, :], axis=2) + + +def myplot(axis, X, Y, Z, xlabel="none", ylabel="none"): frame = axis.contourf(X, Y, Z) plt.colorbar(frame, ax=axis) - if (xlabel != 'none'): + if xlabel != "none": axis.set_xlabel(xlabel) - if (ylabel != 'none'): + if ylabel != "none": axis.set_ylabel(ylabel) - # read time mesh times = np.loadtxt("t.000000.txt") nt = times.size # read spatial mesh mesh = np.loadtxt("mesh.txt", dtype=float) -x = mesh[0,:] -y = mesh[1,:] -z = mesh[2,:] +x = mesh[0, :] +y = mesh[1, :] +z = mesh[2, :] nx = x.size ny = y.size nz = z.size # ensure that the run used exactly 1 or 8 MPI ranks for i in range(9): - if (exists("u.00000" + str(i) + ".txt" ) and - not exists("u.00000" + str(i+1) + ".txt" )): - nprocs = i+1 -if ((nprocs != 1) and (nprocs != 8)): + if exists("u.00000" + str(i) + ".txt") and not exists( + "u.00000" + str(i + 1) + ".txt" + ): + nprocs = i + 1 +if (nprocs != 1) and (nprocs != 8): print("make_plots.py error: run must have used either 1 or 8 MPI ranks") exit() # load data for run -if (nprocs == 1): - u = np.zeros((nt,nx,ny,nz), dtype=float) - v = np.zeros((nt,nx,ny,nz), dtype=float) - w = np.zeros((nt,nx,ny,nz), dtype=float) +if nprocs == 1: + u = np.zeros((nt, nx, ny, nz), dtype=float) + v = np.zeros((nt, nx, ny, nz), dtype=float) + w = np.zeros((nt, nx, ny, nz), dtype=float) udata = np.loadtxt("u.000000.txt") vdata = np.loadtxt("v.000000.txt") wdata = np.loadtxt("w.000000.txt") - if (nt != udata.shape[0]): + if nt != udata.shape[0]: print("make_plots.py error: mesh and data have incompatible sizes") exit() - if (nx*ny*nz != udata.shape[1]): + if nx * ny * nz != udata.shape[1]: print("make_plots.py error: mesh and data have incompatible sizes") exit() for it in range(nt): - u[it,:,:,:] = np.reshape(udata[it,:], (nx,ny,nz), order='C') - v[it,:,:,:] = np.reshape(vdata[it,:], (nx,ny,nz), order='C') - w[it,:,:,:] = np.reshape(wdata[it,:], (nx,ny,nz), order='C') + u[it, :, :, :] = np.reshape(udata[it, :], (nx, ny, nz), order="C") + v[it, :, :, :] = np.reshape(vdata[it, :], (nx, ny, nz), order="C") + w[it, :, :, :] = np.reshape(wdata[it, :], (nx, ny, nz), order="C") else: - u = np.zeros((nt,nx,ny,nz), dtype=float) - v = np.zeros((nt,nx,ny,nz), dtype=float) - w = np.zeros((nt,nx,ny,nz), dtype=float) - nxl = nx//2 - nyl = ny//2 - nzl = nz//2 + u = np.zeros((nt, nx, ny, nz), dtype=float) + v = np.zeros((nt, nx, ny, nz), dtype=float) + w = np.zeros((nt, nx, ny, nz), dtype=float) + nxl = nx // 2 + nyl = ny // 2 + nzl = nz // 2 for ip in range(8): udata = np.loadtxt("u.00000" + str(ip) + ".txt") vdata = np.loadtxt("v.00000" + str(ip) + ".txt") wdata = np.loadtxt("w.00000" + str(ip) + ".txt") - if (nt != udata.shape[0]): + if nt != udata.shape[0]: print("make_plots.py error: mesh and data have incompatible sizes") exit() - if (nxl*nyl*nzl != udata.shape[1]): + if nxl * nyl * nzl != udata.shape[1]: print("make_plots.py error: mesh and data have incompatible sizes") exit() coords = parallel_coords(ip) - ilo = coords[0]*nxl - ihi = (coords[0]+1)*nxl - jlo = coords[1]*nyl - jhi = (coords[1]+1)*nyl - klo = coords[2]*nzl - khi = (coords[2]+1)*nzl + ilo = coords[0] * nxl + ihi = (coords[0] + 1) * nxl + jlo = coords[1] * nyl + jhi = (coords[1] + 1) * nyl + klo = coords[2] * nzl + khi = (coords[2] + 1) * nzl for it in range(nt): - u[it,ilo:ihi,jlo:jhi,klo:khi] = np.reshape(udata[it,:], (nxl,nyl,nzl), order='C') - v[it,ilo:ihi,jlo:jhi,klo:khi] = np.reshape(vdata[it,:], (nxl,nyl,nzl), order='C') - w[it,ilo:ihi,jlo:jhi,klo:khi] = np.reshape(wdata[it,:], (nxl,nyl,nzl), order='C') + u[it, ilo:ihi, jlo:jhi, klo:khi] = np.reshape( + udata[it, :], (nxl, nyl, nzl), order="C" + ) + v[it, ilo:ihi, jlo:jhi, klo:khi] = np.reshape( + vdata[it, :], (nxl, nyl, nzl), order="C" + ) + w[it, ilo:ihi, jlo:jhi, klo:khi] = np.reshape( + wdata[it, :], (nxl, nyl, nzl), order="C" + ) # set meshgrid objects -xy0,xy1 = np.meshgrid(x, y) -yz0,yz1 = np.meshgrid(y, z) -xz0,xz1 = np.meshgrid(x, z) +xy0, xy1 = np.meshgrid(x, y) +yz0, yz1 = np.meshgrid(y, z) +xz0, xz1 = np.meshgrid(x, z) # generate plots sliceidx = 25 tslice = [0, 5, 10] -figsize = (9,7) +figsize = (9, 7) # xy slices at various times plt.figure(1) -fig, ((ax1,ax2,ax3), (ax4,ax5,ax6), (ax7,ax8,ax9)) = plt.subplots(3, 3, sharex=True, sharey=True, figsize=figsize) -myplot(ax1, xy0, xy1, zslice(u,tslice[0],sliceidx), ylabel = 'u') -myplot(ax2, xy0, xy1, zslice(u,tslice[1],sliceidx)) -myplot(ax3, xy0, xy1, zslice(u,tslice[2],sliceidx)) -myplot(ax4, xy0, xy1, zslice(v,tslice[0],sliceidx), ylabel = 'v') -myplot(ax5, xy0, xy1, zslice(v,tslice[1],sliceidx)) -myplot(ax6, xy0, xy1, zslice(v,tslice[2],sliceidx)) -myplot(ax7, xy0, xy1, zslice(w,tslice[0],sliceidx), ylabel = 'w', xlabel = 't = ' + str(times[0])) -myplot(ax8, xy0, xy1, zslice(w,tslice[1],sliceidx), xlabel = 't = ' + str(times[1])) -myplot(ax9, xy0, xy1, zslice(w,tslice[2],sliceidx), xlabel = 't = ' + str(times[2])) -plt.savefig('xy-slices.png') +fig, ((ax1, ax2, ax3), (ax4, ax5, ax6), (ax7, ax8, ax9)) = plt.subplots( + 3, 3, sharex=True, sharey=True, figsize=figsize +) +myplot(ax1, xy0, xy1, zslice(u, tslice[0], sliceidx), ylabel="u") +myplot(ax2, xy0, xy1, zslice(u, tslice[1], sliceidx)) +myplot(ax3, xy0, xy1, zslice(u, tslice[2], sliceidx)) +myplot(ax4, xy0, xy1, zslice(v, tslice[0], sliceidx), ylabel="v") +myplot(ax5, xy0, xy1, zslice(v, tslice[1], sliceidx)) +myplot(ax6, xy0, xy1, zslice(v, tslice[2], sliceidx)) +myplot( + ax7, + xy0, + xy1, + zslice(w, tslice[0], sliceidx), + ylabel="w", + xlabel="t = " + str(times[0]), +) +myplot(ax8, xy0, xy1, zslice(w, tslice[1], sliceidx), xlabel="t = " + str(times[1])) +myplot(ax9, xy0, xy1, zslice(w, tslice[2], sliceidx), xlabel="t = " + str(times[2])) +plt.savefig("xy-slices.png") # yz slices at various times plt.figure(2) -fig, ((ax1,ax2,ax3), (ax4,ax5,ax6), (ax7,ax8,ax9)) = plt.subplots(3, 3, sharex=True, sharey=True, figsize=figsize) -myplot(ax1, yz0, yz1, xslice(u,tslice[0],sliceidx), ylabel = 'u') -myplot(ax2, yz0, yz1, xslice(u,tslice[1],sliceidx)) -myplot(ax3, yz0, yz1, xslice(u,tslice[2],sliceidx)) -myplot(ax4, yz0, yz1, xslice(v,tslice[0],sliceidx), ylabel = 'v') -myplot(ax5, yz0, yz1, xslice(v,tslice[1],sliceidx)) -myplot(ax6, yz0, yz1, xslice(v,tslice[2],sliceidx)) -myplot(ax7, yz0, yz1, xslice(w,tslice[0],sliceidx), ylabel = 'w', xlabel = 't = ' + str(times[0])) -myplot(ax8, yz0, yz1, xslice(w,tslice[1],sliceidx), xlabel = 't = ' + str(times[1])) -myplot(ax9, yz0, yz1, xslice(w,tslice[2],sliceidx), xlabel = 't = ' + str(times[2])) -plt.savefig('yz-slices.png') +fig, ((ax1, ax2, ax3), (ax4, ax5, ax6), (ax7, ax8, ax9)) = plt.subplots( + 3, 3, sharex=True, sharey=True, figsize=figsize +) +myplot(ax1, yz0, yz1, xslice(u, tslice[0], sliceidx), ylabel="u") +myplot(ax2, yz0, yz1, xslice(u, tslice[1], sliceidx)) +myplot(ax3, yz0, yz1, xslice(u, tslice[2], sliceidx)) +myplot(ax4, yz0, yz1, xslice(v, tslice[0], sliceidx), ylabel="v") +myplot(ax5, yz0, yz1, xslice(v, tslice[1], sliceidx)) +myplot(ax6, yz0, yz1, xslice(v, tslice[2], sliceidx)) +myplot( + ax7, + yz0, + yz1, + xslice(w, tslice[0], sliceidx), + ylabel="w", + xlabel="t = " + str(times[0]), +) +myplot(ax8, yz0, yz1, xslice(w, tslice[1], sliceidx), xlabel="t = " + str(times[1])) +myplot(ax9, yz0, yz1, xslice(w, tslice[2], sliceidx), xlabel="t = " + str(times[2])) +plt.savefig("yz-slices.png") # xz slices at various times plt.figure(3) -fig, ((ax1,ax2,ax3), (ax4,ax5,ax6), (ax7,ax8,ax9)) = plt.subplots(3, 3, sharex=True, sharey=True, figsize=figsize) -myplot(ax1, xz0, xz1, yslice(u,tslice[0],sliceidx), ylabel ='u') -myplot(ax2, xz0, xz1, yslice(u,tslice[1],sliceidx)) -myplot(ax3, xz0, xz1, yslice(u,tslice[2],sliceidx)) -myplot(ax4, xz0, xz1, yslice(v,tslice[0],sliceidx), ylabel = 'v') -myplot(ax5, xz0, xz1, yslice(v,tslice[1],sliceidx)) -myplot(ax6, xz0, xz1, yslice(v,tslice[2],sliceidx)) -myplot(ax7, xz0, xz1, yslice(w,tslice[0],sliceidx), ylabel= 'w', xlabel = 't = ' + str(times[0])) -myplot(ax8, xz0, xz1, yslice(w,tslice[1],sliceidx), xlabel ='t = ' + str(times[1])) -myplot(ax9, xz0, xz1, yslice(w,tslice[2],sliceidx), xlabel = 't = ' + str(times[2])) -plt.savefig('xz-slices.png') +fig, ((ax1, ax2, ax3), (ax4, ax5, ax6), (ax7, ax8, ax9)) = plt.subplots( + 3, 3, sharex=True, sharey=True, figsize=figsize +) +myplot(ax1, xz0, xz1, yslice(u, tslice[0], sliceidx), ylabel="u") +myplot(ax2, xz0, xz1, yslice(u, tslice[1], sliceidx)) +myplot(ax3, xz0, xz1, yslice(u, tslice[2], sliceidx)) +myplot(ax4, xz0, xz1, yslice(v, tslice[0], sliceidx), ylabel="v") +myplot(ax5, xz0, xz1, yslice(v, tslice[1], sliceidx)) +myplot(ax6, xz0, xz1, yslice(v, tslice[2], sliceidx)) +myplot( + ax7, + xz0, + xz1, + yslice(w, tslice[0], sliceidx), + ylabel="w", + xlabel="t = " + str(times[0]), +) +myplot(ax8, xz0, xz1, yslice(w, tslice[1], sliceidx), xlabel="t = " + str(times[1])) +myplot(ax9, xz0, xz1, yslice(w, tslice[2], sliceidx), xlabel="t = " + str(times[2])) +plt.savefig("xz-slices.png") # xy projection at various times plt.figure(4) -fig, ((ax1,ax2,ax3), (ax4,ax5,ax6), (ax7,ax8,ax9)) = plt.subplots(3, 3, sharex=True, sharey=True, figsize=figsize) -myplot(ax1, xy0, xy1, zproj(u,tslice[0]), ylabel = 'u') -myplot(ax2, xy0, xy1, zproj(u,tslice[1])) -myplot(ax3, xy0, xy1, zproj(u,tslice[2])) -myplot(ax4, xy0, xy1, zproj(v,tslice[0]), ylabel = 'v') -myplot(ax5, xy0, xy1, zproj(v,tslice[1])) -myplot(ax6, xy0, xy1, zproj(v,tslice[2])) -myplot(ax7, xy0, xy1, zproj(w,tslice[0]), ylabel = 'w', xlabel = 't = ' + str(times[0])) -myplot(ax8, xy0, xy1, zproj(w,tslice[1]), xlabel = 't = ' + str(times[1])) -myplot(ax9, xy0, xy1, zproj(w,tslice[2]), xlabel = 't = ' + str(times[2])) -plt.savefig('xy-projections.png') +fig, ((ax1, ax2, ax3), (ax4, ax5, ax6), (ax7, ax8, ax9)) = plt.subplots( + 3, 3, sharex=True, sharey=True, figsize=figsize +) +myplot(ax1, xy0, xy1, zproj(u, tslice[0]), ylabel="u") +myplot(ax2, xy0, xy1, zproj(u, tslice[1])) +myplot(ax3, xy0, xy1, zproj(u, tslice[2])) +myplot(ax4, xy0, xy1, zproj(v, tslice[0]), ylabel="v") +myplot(ax5, xy0, xy1, zproj(v, tslice[1])) +myplot(ax6, xy0, xy1, zproj(v, tslice[2])) +myplot(ax7, xy0, xy1, zproj(w, tslice[0]), ylabel="w", xlabel="t = " + str(times[0])) +myplot(ax8, xy0, xy1, zproj(w, tslice[1]), xlabel="t = " + str(times[1])) +myplot(ax9, xy0, xy1, zproj(w, tslice[2]), xlabel="t = " + str(times[2])) +plt.savefig("xy-projections.png") # yz projection at various times fig = plt.figure(5) -fig, ((ax1,ax2,ax3), (ax4,ax5,ax6), (ax7,ax8,ax9)) = plt.subplots(3, 3, sharex=True, sharey=True, figsize=figsize) -myplot(ax1, yz0, yz1, xproj(u,tslice[0]), ylabel = 'u') -myplot(ax2, yz0, yz1, xproj(u,tslice[1])) -myplot(ax3, yz0, yz1, xproj(u,tslice[2])) -myplot(ax4, yz0, yz1, xproj(v,tslice[0]), ylabel = 'v') -myplot(ax5, yz0, yz1, xproj(v,tslice[1])) -myplot(ax6, yz0, yz1, xproj(v,tslice[2])) -myplot(ax7, yz0, yz1, xproj(w,tslice[0]), ylabel = 'w', xlabel = 't = ' + str(times[0])) -myplot(ax8, yz0, yz1, xproj(w,tslice[1]), xlabel = 't = ' + str(times[1])) -myplot(ax9, yz0, yz1, xproj(w,tslice[2]), xlabel = 't = ' + str(times[2])) -plt.savefig('yz-projections.png') +fig, ((ax1, ax2, ax3), (ax4, ax5, ax6), (ax7, ax8, ax9)) = plt.subplots( + 3, 3, sharex=True, sharey=True, figsize=figsize +) +myplot(ax1, yz0, yz1, xproj(u, tslice[0]), ylabel="u") +myplot(ax2, yz0, yz1, xproj(u, tslice[1])) +myplot(ax3, yz0, yz1, xproj(u, tslice[2])) +myplot(ax4, yz0, yz1, xproj(v, tslice[0]), ylabel="v") +myplot(ax5, yz0, yz1, xproj(v, tslice[1])) +myplot(ax6, yz0, yz1, xproj(v, tslice[2])) +myplot(ax7, yz0, yz1, xproj(w, tslice[0]), ylabel="w", xlabel="t = " + str(times[0])) +myplot(ax8, yz0, yz1, xproj(w, tslice[1]), xlabel="t = " + str(times[1])) +myplot(ax9, yz0, yz1, xproj(w, tslice[2]), xlabel="t = " + str(times[2])) +plt.savefig("yz-projections.png") # xz projection at various times fig = plt.figure(6) -fig, ((ax1,ax2,ax3), (ax4,ax5,ax6), (ax7,ax8,ax9)) = plt.subplots(3, 3, sharex=True, sharey=True, figsize=figsize) -myplot(ax1, xz0, xz1, yproj(u,tslice[0]), ylabel = 'u') -myplot(ax2, xz0, xz1, yproj(u,tslice[1])) -myplot(ax3, xz0, xz1, yproj(u,tslice[2])) -myplot(ax4, xz0, xz1, yproj(v,tslice[0]), ylabel = 'v') -myplot(ax5, xz0, xz1, yproj(v,tslice[1])) -myplot(ax6, xz0, xz1, yproj(v,tslice[2])) -myplot(ax7, xz0, xz1, yproj(w,tslice[0]), ylabel = 'w', xlabel = 't = ' + str(times[0])) -myplot(ax8, xz0, xz1, yproj(w,tslice[1]), xlabel = 't = ' + str(times[1])) -myplot(ax9, xz0, xz1, yproj(w,tslice[2]), xlabel = 't = ' + str(times[2])) -plt.savefig('xz-projections.png') - -#plt.show() +fig, ((ax1, ax2, ax3), (ax4, ax5, ax6), (ax7, ax8, ax9)) = plt.subplots( + 3, 3, sharex=True, sharey=True, figsize=figsize +) +myplot(ax1, xz0, xz1, yproj(u, tslice[0]), ylabel="u") +myplot(ax2, xz0, xz1, yproj(u, tslice[1])) +myplot(ax3, xz0, xz1, yproj(u, tslice[2])) +myplot(ax4, xz0, xz1, yproj(v, tslice[0]), ylabel="v") +myplot(ax5, xz0, xz1, yproj(v, tslice[1])) +myplot(ax6, xz0, xz1, yproj(v, tslice[2])) +myplot(ax7, xz0, xz1, yproj(w, tslice[0]), ylabel="w", xlabel="t = " + str(times[0])) +myplot(ax8, xz0, xz1, yproj(w, tslice[1]), xlabel="t = " + str(times[1])) +myplot(ax9, xz0, xz1, yproj(w, tslice[2]), xlabel="t = " + str(times[2])) +plt.savefig("xz-projections.png") + +# plt.show() plt.close() ##### end of script ##### diff --git a/benchmarks/advection_reaction_3D/scripts/pickle_solution_output.py b/benchmarks/advection_reaction_3D/scripts/pickle_solution_output.py index 407c34921a..a51fade40f 100755 --- a/benchmarks/advection_reaction_3D/scripts/pickle_solution_output.py +++ b/benchmarks/advection_reaction_3D/scripts/pickle_solution_output.py @@ -19,39 +19,66 @@ import numpy as np # load mesh data file -mesh = np.loadtxt('mesh.txt', dtype=np.double) +mesh = np.loadtxt("mesh.txt", dtype=np.double) # X,Y,Z = np.meshgrid(mesh[0,:], mesh[1,:], mesh[2,:]) # calculate h -hx = mesh[0,1] - mesh[0,0] -hy = mesh[1,1] - mesh[1,0] -hz = mesh[2,1] - mesh[2,0] -nx = len(mesh[0,:]) -ny = len(mesh[1,:]) -nz = len(mesh[2,:]) +hx = mesh[0, 1] - mesh[0, 0] +hy = mesh[1, 1] - mesh[1, 0] +hz = mesh[2, 1] - mesh[2, 0] +nx = len(mesh[0, :]) +ny = len(mesh[1, :]) +nz = len(mesh[2, :]) print("nx, ny, nz = %d, %d, %d" % (nx, ny, nz)) print("hx, hy, hz = %g, %g, %g" % (hx, hy, hz)) # load output time file -times = np.loadtxt('t.000000.txt', dtype=np.double) +times = np.loadtxt("t.000000.txt", dtype=np.double) # load solution data files -ufiles = glob.glob('u.' + ('[0-9]'*6) + '.txt'); ufiles.sort() -vfiles = glob.glob('v.' + ('[0-9]'*6) + '.txt'); vfiles.sort() -wfiles = glob.glob('w.' + ('[0-9]'*6) + '.txt'); wfiles.sort() +ufiles = glob.glob("u." + ("[0-9]" * 6) + ".txt") +ufiles.sort() +vfiles = glob.glob("v." + ("[0-9]" * 6) + ".txt") +vfiles.sort() +wfiles = glob.glob("w." + ("[0-9]" * 6) + ".txt") +wfiles.sort() udata = [] vdata = [] wdata = [] sys.stdout.write("reading 1/%d...\r" % len(ufiles)) sys.stdout.flush() -for idx in range(0,len(ufiles)): - sys.stdout.write("reading %d/%d...\r" % (idx+1,len(ufiles))) +for idx in range(0, len(ufiles)): + sys.stdout.write("reading %d/%d...\r" % (idx + 1, len(ufiles))) sys.stdout.flush() - udata.append(pd.read_csv(ufiles[idx], header=None, delimiter=' ', skipinitialspace=True, dtype=np.double)) - vdata.append(pd.read_csv(vfiles[idx], header=None, delimiter=' ', skipinitialspace=True, dtype=np.double)) - wdata.append(pd.read_csv(wfiles[idx], header=None, delimiter=' ', skipinitialspace=True, dtype=np.double)) + udata.append( + pd.read_csv( + ufiles[idx], + header=None, + delimiter=" ", + skipinitialspace=True, + dtype=np.double, + ) + ) + vdata.append( + pd.read_csv( + vfiles[idx], + header=None, + delimiter=" ", + skipinitialspace=True, + dtype=np.double, + ) + ) + wdata.append( + pd.read_csv( + wfiles[idx], + header=None, + delimiter=" ", + skipinitialspace=True, + dtype=np.double, + ) + ) sys.stdout.write("\n") sys.stdout.flush() @@ -69,5 +96,6 @@ # save data to pickle print("saving...") -np.savez_compressed('output-with-h-%.2e.npz' % hx, t=times, u=udata, v=vdata, w=wdata, mesh=mesh) - +np.savez_compressed( + "output-with-h-%.2e.npz" % hx, t=times, u=udata, v=vdata, w=wdata, mesh=mesh +) diff --git a/benchmarks/diffusion_2D/CMakeLists.txt b/benchmarks/diffusion_2D/CMakeLists.txt index f64d3e90f3..f26f7397c2 100644 --- a/benchmarks/diffusion_2D/CMakeLists.txt +++ b/benchmarks/diffusion_2D/CMakeLists.txt @@ -12,13 +12,13 @@ # SUNDIALS Copyright End # ------------------------------------------------------------------------------ -if(BUILD_ARKODE OR BUILD_CVODE OR BUILD_IDA) +if(BUILD_ARKODE + OR BUILD_CVODE + OR BUILD_IDA) # Shared sources - set(shared_sources - diffusion_2D.hpp - diffusion_2D.cpp - preconditioner_jacobi.cpp) + set(shared_sources diffusion_2D.hpp diffusion_2D.cpp + preconditioner_jacobi.cpp) # Benchmark prefix set(benchmark_prefix ${SUNDIALS_SOURCE_DIR}/benchmarks/diffusion_2D/) @@ -33,7 +33,6 @@ if(BUILD_ARKODE OR BUILD_CVODE OR BUILD_IDA) add_subdirectory(mpi_gpu) endif() - install(FILES README.md - DESTINATION "${BENCHMARKS_INSTALL_PATH}/diffusion_2D") + install(FILES README.md DESTINATION "${BENCHMARKS_INSTALL_PATH}/diffusion_2D") endif() diff --git a/benchmarks/diffusion_2D/diffusion_2D.cpp b/benchmarks/diffusion_2D/diffusion_2D.cpp index bd18cf1dfb..2a64400938 100644 --- a/benchmarks/diffusion_2D/diffusion_2D.cpp +++ b/benchmarks/diffusion_2D/diffusion_2D.cpp @@ -11,7 +11,7 @@ * SPDX-License-Identifier: BSD-3-Clause * SUNDIALS Copyright End * ----------------------------------------------------------------------------- - * Shared implementaiton file for 2D diffusion benchmark problem + * Shared implementation file for 2D diffusion benchmark problem * ---------------------------------------------------------------------------*/ #include "diffusion_2D.hpp" diff --git a/benchmarks/diffusion_2D/diffusion_2D.hpp b/benchmarks/diffusion_2D/diffusion_2D.hpp index 1ebb3394aa..a27c004e63 100644 --- a/benchmarks/diffusion_2D/diffusion_2D.hpp +++ b/benchmarks/diffusion_2D/diffusion_2D.hpp @@ -159,7 +159,7 @@ struct UserData sunrealtype* Ssend = NULL; sunrealtype* Nsend = NULL; - // Send requests for neighor exchange + // Send requests for neighbor exchange MPI_Request reqSW; MPI_Request reqSE; MPI_Request reqSS; @@ -192,7 +192,7 @@ struct UserData struct UserOutput { - // Ouput variables + // Output variables int output = 1; // 0 = no output, 1 = stats output, 2 = output to disk int nout = 20; // number of output times N_Vector error = NULL; // error vector diff --git a/benchmarks/diffusion_2D/main_arkode.cpp b/benchmarks/diffusion_2D/main_arkode.cpp index c35bf9e9c7..e569cffec2 100644 --- a/benchmarks/diffusion_2D/main_arkode.cpp +++ b/benchmarks/diffusion_2D/main_arkode.cpp @@ -15,7 +15,10 @@ * ---------------------------------------------------------------------------*/ #include "arkode/arkode_arkstep.h" +#include "arkode/arkode_lsrkstep.h" #include "diffusion_2D.hpp" +#include "sunadaptcontroller/sunadaptcontroller_imexgus.h" +#include "sunadaptcontroller/sunadaptcontroller_soderlind.h" struct UserOptions { @@ -28,6 +31,8 @@ struct UserOptions int maxsteps = 0; // max steps between outputs int onestep = 0; // one step mode, number of steps bool linear = true; // linearly implicit RHS + bool implicit = true; // implicit (ARKStep) vs explicit STS (LSRKStep) + ARKODE_LSRKMethodType lsrkmethod = ARKODE_LSRK_RKC_2; // LSRK method type // Linear solver and preconditioner settings std::string ls = "cg"; // linear solver to use @@ -43,6 +48,14 @@ struct UserOptions void print(); }; +// ----------------------------------------------------------------------------- +// LSRKStep-specific dominant eigenvalue function prototype +// ----------------------------------------------------------------------------- + +static int dom_eig(sunrealtype t, N_Vector y, N_Vector fn, sunrealtype* lambdaR, + sunrealtype* lambdaI, void* user_data, N_Vector temp1, + N_Vector temp2, N_Vector temp3); + // ----------------------------------------------------------------------------- // Main Program // ----------------------------------------------------------------------------- @@ -58,8 +71,8 @@ int main(int argc, char* argv[]) // Create SUNDIALS context MPI_Comm comm = MPI_COMM_WORLD; - SUNContext ctx = NULL; - SUNProfiler prof = NULL; + SUNContext ctx = nullptr; + SUNProfiler prof = nullptr; flag = SUNContext_Create(comm, &ctx); if (check_flag(&flag, "SUNContextCreate", 1)) { return 1; } @@ -109,6 +122,17 @@ int main(int argc, char* argv[]) return 1; } + // Return with error on unsupported LSRK method type + if (!uopts.implicit) + { + if ((uopts.lsrkmethod != ARKODE_LSRK_RKC_2) && + (uopts.lsrkmethod != ARKODE_LSRK_RKL_2)) + { + cerr << "ERROR: illegal lsrkmethod" << endl; + return 1; + } + } + // ----------------------------- // Setup parallel decomposition // ----------------------------- @@ -152,12 +176,8 @@ int main(int argc, char* argv[]) if (check_flag((void*)(uout.error), "N_VClone", 0)) { return 1; } } - // --------------------- - // Create linear solver - // --------------------- - - // Create linear solver - SUNLinearSolver LS = NULL; + // Set up implicit solver, if applicable + SUNLinearSolver LS = nullptr; SUNMatrix A = nullptr; #if defined(USE_SUPERLU_DIST) // SuperLU-DIST objects @@ -172,77 +192,96 @@ int main(int argc, char* argv[]) sunindextype* A_col_idxs = nullptr; sunindextype* A_row_ptrs = nullptr; #endif - - int prectype = (uopts.preconditioning) ? SUN_PREC_RIGHT : SUN_PREC_NONE; - - if (uopts.ls == "cg") - { - LS = SUNLinSol_PCG(u, prectype, uopts.liniters, ctx); - if (check_flag((void*)LS, "SUNLinSol_PCG", 0)) { return 1; } - } - else if (uopts.ls == "gmres") + if (uopts.implicit) { - LS = SUNLinSol_SPGMR(u, prectype, uopts.liniters, ctx); - if (check_flag((void*)LS, "SUNLinSol_SPGMR", 0)) { return 1; } - } - else - { -#if defined(USE_SUPERLU_DIST) - // Initialize SuperLU-DIST grid - superlu_gridinit(udata.comm_c, udata.npx, udata.npy, &grid); - - // Create arrays for CSR matrix: data, column indices, and row pointers - sunindextype nnz_loc = 5 * udata.nodes_loc; - - A_data = (sunrealtype*)malloc(nnz_loc * sizeof(sunrealtype)); - if (check_flag((void*)A_data, "malloc Adata", 0)) return 1; - - A_col_idxs = (sunindextype*)malloc(nnz_loc * sizeof(sunindextype)); - if (check_flag((void*)A_col_idxs, "malloc Acolind", 0)) return 1; + // --------------------- + // Create linear solver + // --------------------- - A_row_ptrs = - (sunindextype*)malloc((udata.nodes_loc + 1) * sizeof(sunindextype)); - if (check_flag((void*)A_row_ptrs, "malloc Arowptr", 0)) return 1; + // Create linear solver - // Create and initialize SuperLU_DIST structures - dCreate_CompRowLoc_Matrix_dist(&A_super, udata.nodes, udata.nodes, nnz_loc, - udata.nodes_loc, 0, A_data, A_col_idxs, - A_row_ptrs, SLU_NR_loc, SLU_D, SLU_GE); - dScalePermstructInit(udata.nodes, udata.nodes, &A_scaleperm); - dLUstructInit(udata.nodes, &A_lu); - PStatInit(&A_stat); - set_default_options_dist(&A_opts); - A_opts.PrintStat = NO; + int prectype = (uopts.preconditioning) ? SUN_PREC_RIGHT : SUN_PREC_NONE; - // SUNDIALS structures - A = SUNMatrix_SLUNRloc(&A_super, &grid, ctx); - if (check_flag((void*)A, "SUNMatrix_SLUNRloc", 0)) return 1; - - LS = SUNLinSol_SuperLUDIST(u, A, &grid, &A_lu, &A_scaleperm, &A_solve, - &A_stat, &A_opts, ctx); - if (check_flag((void*)LS, "SUNLinSol_SuperLUDIST", 0)) return 1; - - uopts.preconditioning = false; + if (uopts.ls == "cg") + { + LS = SUNLinSol_PCG(u, prectype, uopts.liniters, ctx); + if (check_flag((void*)LS, "SUNLinSol_PCG", 0)) { return 1; } + } + else if (uopts.ls == "gmres") + { + LS = SUNLinSol_SPGMR(u, prectype, uopts.liniters, ctx); + if (check_flag((void*)LS, "SUNLinSol_SPGMR", 0)) { return 1; } + } + else + { +#if defined(USE_SUPERLU_DIST) + // Initialize SuperLU-DIST grid + superlu_gridinit(udata.comm_c, udata.npx, udata.npy, &grid); + + // Create arrays for CSR matrix: data, column indices, and row pointers + sunindextype nnz_loc = 5 * udata.nodes_loc; + + A_data = (sunrealtype*)malloc(nnz_loc * sizeof(sunrealtype)); + if (check_flag((void*)A_data, "malloc Adata", 0)) return 1; + + A_col_idxs = (sunindextype*)malloc(nnz_loc * sizeof(sunindextype)); + if (check_flag((void*)A_col_idxs, "malloc Acolind", 0)) return 1; + + A_row_ptrs = + (sunindextype*)malloc((udata.nodes_loc + 1) * sizeof(sunindextype)); + if (check_flag((void*)A_row_ptrs, "malloc Arowptr", 0)) return 1; + + // Create and initialize SuperLU_DIST structures + dCreate_CompRowLoc_Matrix_dist(&A_super, udata.nodes, udata.nodes, + nnz_loc, udata.nodes_loc, 0, A_data, + A_col_idxs, A_row_ptrs, SLU_NR_loc, + SLU_D, SLU_GE); + dScalePermstructInit(udata.nodes, udata.nodes, &A_scaleperm); + dLUstructInit(udata.nodes, &A_lu); + PStatInit(&A_stat); + set_default_options_dist(&A_opts); + A_opts.PrintStat = NO; + + // SUNDIALS structures + A = SUNMatrix_SLUNRloc(&A_super, &grid, ctx); + if (check_flag((void*)A, "SUNMatrix_SLUNRloc", 0)) return 1; + + LS = SUNLinSol_SuperLUDIST(u, A, &grid, &A_lu, &A_scaleperm, &A_solve, + &A_stat, &A_opts, ctx); + if (check_flag((void*)LS, "SUNLinSol_SuperLUDIST", 0)) return 1; + + uopts.preconditioning = false; #else - std::cerr << "ERROR: Benchmark was not built with SuperLU_DIST enabled\n"; - return 1; + std::cerr + << "ERROR: Benchmark was not built with SuperLU_DIST enabled\n"; + return 1; #endif - } + } - // Allocate preconditioner workspace - if (uopts.preconditioning) - { - udata.diag = N_VClone(u); - if (check_flag((void*)(udata.diag), "N_VClone", 0)) { return 1; } + // Allocate preconditioner workspace + if (uopts.preconditioning) + { + udata.diag = N_VClone(u); + if (check_flag((void*)(udata.diag), "N_VClone", 0)) { return 1; } + } } - // -------------- - // Setup ARKStep - // -------------- + // ---------------------- + // Setup ARKStep/LSRKStep + // ---------------------- // Create integrator - void* arkode_mem = ARKStepCreate(NULL, diffusion, ZERO, u, ctx); - if (check_flag((void*)arkode_mem, "ARKStepCreate", 0)) { return 1; } + void* arkode_mem = nullptr; + if (uopts.implicit) + { + arkode_mem = ARKStepCreate(nullptr, diffusion, ZERO, u, ctx); + if (check_flag((void*)arkode_mem, "ARKStepCreate", 0)) { return 1; } + } + else + { + arkode_mem = LSRKStepCreateSTS(diffusion, ZERO, u, ctx); + if (check_flag((void*)arkode_mem, "LSRKStepCreateSTS", 0)) { return 1; } + } // Specify tolerances flag = ARKodeSStolerances(arkode_mem, uopts.rtol, uopts.atol); @@ -252,38 +291,60 @@ int main(int argc, char* argv[]) flag = ARKodeSetUserData(arkode_mem, (void*)&udata); if (check_flag(&flag, "ARKodeSetUserData", 1)) { return 1; } - // Attach linear solver - flag = ARKodeSetLinearSolver(arkode_mem, LS, A); - if (check_flag(&flag, "ARKodeSetLinearSolver", 1)) { return 1; } + // Configure implicit solver + if (uopts.implicit) + { + // Attach linear solver + flag = ARKodeSetLinearSolver(arkode_mem, LS, A); + if (check_flag(&flag, "ARKodeSetLinearSolver", 1)) { return 1; } #if defined(USE_SUPERLU_DIST) - if (uopts.ls == "sludist") - { - ARKodeSetJacFn(arkode_mem, diffusion_jac); - if (check_flag(&flag, "ARKodeSetJacFn", 1)) return 1; - } + if (uopts.ls == "sludist") + { + ARKodeSetJacFn(arkode_mem, diffusion_jac); + if (check_flag(&flag, "ARKodeSetJacFn", 1)) return 1; + } #endif - if (uopts.preconditioning) - { - // Attach preconditioner - flag = ARKodeSetPreconditioner(arkode_mem, PSetup, PSolve); - if (check_flag(&flag, "ARKodeSetPreconditioner", 1)) { return 1; } + if (uopts.preconditioning) + { + // Attach preconditioner + flag = ARKodeSetPreconditioner(arkode_mem, PSetup, PSolve); + if (check_flag(&flag, "ARKodeSetPreconditioner", 1)) { return 1; } - // Set linear solver setup frequency (update preconditioner) - flag = ARKodeSetLSetupFrequency(arkode_mem, uopts.msbp); - if (check_flag(&flag, "ARKodeSetLSetupFrequency", 1)) { return 1; } - } + // Set linear solver setup frequency (update preconditioner) + flag = ARKodeSetLSetupFrequency(arkode_mem, uopts.msbp); + if (check_flag(&flag, "ARKodeSetLSetupFrequency", 1)) { return 1; } + } + + // Set linear solver tolerance factor + flag = ARKodeSetEpsLin(arkode_mem, uopts.epslin); + if (check_flag(&flag, "ARKodeSetEpsLin", 1)) { return 1; } + + // Select method order + flag = ARKodeSetOrder(arkode_mem, uopts.order); + if (check_flag(&flag, "ARKodeSetOrder", 1)) { return 1; } - // Set linear solver tolerance factor - flag = ARKodeSetEpsLin(arkode_mem, uopts.epslin); - if (check_flag(&flag, "ARKodeSetEpsLin", 1)) { return 1; } + // Specify linearly implicit non-time-dependent RHS + if (uopts.linear) + { + flag = ARKodeSetLinear(arkode_mem, 0); + if (check_flag(&flag, "ARKodeSetLinear", 1)) { return 1; } + } + } + else // Configure explicit STS solver + { + // Select LSRK method + flag = LSRKStepSetSTSMethod(arkode_mem, uopts.lsrkmethod); + if (check_flag(&flag, "LSRKStepSetSTSMethod", 1)) { return 1; } - // Select method order - flag = ARKodeSetOrder(arkode_mem, uopts.order); - if (check_flag(&flag, "ARKodeSetOrder", 1)) { return 1; } + // Provide dominant eigenvalue function + flag = LSRKStepSetDomEigFn(arkode_mem, dom_eig); + if (check_flag(&flag, "LSRKStepSetDomEigFn", 1)) { return 1; } + } // Set fixed step size or adaptivity method + SUNAdaptController C = nullptr; if (uopts.hfixed > ZERO) { flag = ARKodeSetFixedStep(arkode_mem, uopts.hfixed); @@ -291,16 +352,17 @@ int main(int argc, char* argv[]) } else { - flag = ARKStepSetAdaptivityMethod(arkode_mem, uopts.controller, SUNTRUE, - SUNFALSE, NULL); - if (check_flag(&flag, "ARKStepSetAdaptivityMethod", 1)) { return 1; } - } - - // Specify linearly implicit non-time-dependent RHS - if (uopts.linear) - { - flag = ARKodeSetLinear(arkode_mem, 0); - if (check_flag(&flag, "ARKodeSetLinear", 1)) { return 1; } + switch (uopts.controller) + { + case (ARK_ADAPT_PID): C = SUNAdaptController_PID(ctx); break; + case (ARK_ADAPT_PI): C = SUNAdaptController_PI(ctx); break; + case (ARK_ADAPT_I): C = SUNAdaptController_I(ctx); break; + case (ARK_ADAPT_EXP_GUS): C = SUNAdaptController_ExpGus(ctx); break; + case (ARK_ADAPT_IMP_GUS): C = SUNAdaptController_ImpGus(ctx); break; + case (ARK_ADAPT_IMEX_GUS): C = SUNAdaptController_ImExGus(ctx); break; + } + flag = ARKodeSetAdaptController(arkode_mem, C); + if (check_flag(&flag, "ARKodeSetAdaptController", 1)) { return 1; } } // Set max steps between outputs @@ -329,7 +391,7 @@ int main(int argc, char* argv[]) sunrealtype dTout = udata.tf / uout.nout; sunrealtype tout = dTout; - // Inital output + // Initial output flag = uout.open(&udata); if (check_flag(&flag, "UserOutput::open", 1)) { return 1; } @@ -377,22 +439,26 @@ int main(int argc, char* argv[]) // Free MPI Cartesian communicator MPI_Comm_free(&(udata.comm_c)); + (void)SUNAdaptController_Destroy(C); // Free timestep adaptivity controller ARKodeFree(&arkode_mem); - SUNLinSolFree(LS); + if (uopts.implicit) + { + SUNLinSolFree(LS); - // Free the SuperLU_DIST structures (also frees user allocated arrays - // A_data, A_col_idxs, and A_row_ptrs) + // Free the SuperLU_DIST structures (also frees user allocated arrays + // A_data, A_col_idxs, and A_row_ptrs) #if defined(USE_SUPERLU_DIST) - if (uopts.ls == "sludist") - { - PStatFree(&A_stat); - dScalePermstructFree(&A_scaleperm); - dLUstructFree(&A_lu); - Destroy_CompRowLoc_Matrix_dist(&A_super); - superlu_gridexit(&grid); - } + if (uopts.ls == "sludist") + { + PStatFree(&A_stat); + dScalePermstructFree(&A_scaleperm); + dLUstructFree(&A_lu); + Destroy_CompRowLoc_Matrix_dist(&A_super); + superlu_gridexit(&grid); + } #endif + } // Free vectors #if defined(USE_HIP) || defined(USE_CUDA) @@ -409,6 +475,26 @@ int main(int argc, char* argv[]) return 0; } +// ----------------------------------------------------------------------------- +// Dominant eigenvalue estimation function +// ----------------------------------------------------------------------------- + +static int dom_eig(sunrealtype t, N_Vector y, N_Vector fn, sunrealtype* lambdaR, + sunrealtype* lambdaI, void* user_data, N_Vector temp1, + N_Vector temp2, N_Vector temp3) +{ + // Access problem data + UserData* udata = (UserData*)user_data; + + // Fill in spectral radius value + *lambdaR = -SUN_RCONST(8.0) * std::max(udata->kx / udata->dx / udata->dx, + udata->ky / udata->dy / udata->dy); + *lambdaI = SUN_RCONST(0.0); + + // return with success + return 0; +} + // ----------------------------------------------------------------------------- // UserOptions Helper functions // ----------------------------------------------------------------------------- @@ -445,6 +531,13 @@ int UserOptions::parse_args(vector& args, bool outproc) args.erase(it, it + 2); } + it = find(args.begin(), args.end(), "--explicitSTS"); + if (it != args.end()) + { + implicit = false; + args.erase(it); + } + it = find(args.begin(), args.end(), "--order"); if (it != args.end()) { @@ -452,6 +545,13 @@ int UserOptions::parse_args(vector& args, bool outproc) args.erase(it, it + 2); } + it = find(args.begin(), args.end(), "--lsrkmethod"); + if (it != args.end()) + { + lsrkmethod = (ARKODE_LSRKMethodType)stoi(*(it + 1)); + args.erase(it, it + 2); + } + it = find(args.begin(), args.end(), "--controller"); if (it != args.end()) { @@ -524,17 +624,23 @@ void UserOptions::help() cout << endl; cout << "Integrator command line options:" << endl; cout << " --rtol : relative tolerance" << endl; - cout << " --atol : absoltue tolerance" << endl; + cout << " --atol : absolute tolerance" << endl; + cout << " --controller : time step adaptivity controller" << endl; + cout << " --fixedstep : used fixed step size" << endl; + cout << " --explicitSTS : use LSRKStep (instead of ARKStep)" << endl; + cout << endl; + cout << "Implicit (ARKStep) solver command line options:" << endl; cout << " --nonlinear : disable linearly implicit flag" << endl; cout << " --order : method order" << endl; - cout << " --fixedstep : used fixed step size" << endl; - cout << " --controller : time step adaptivity controller" << endl; cout << " --ls : linear solver" << endl; cout << " --lsinfo : output residual history" << endl; cout << " --liniters : max number of iterations" << endl; cout << " --epslin : linear tolerance factor" << endl; cout << " --noprec : disable preconditioner" << endl; cout << " --msbp : max steps between prec setups" << endl; + cout << endl; + cout << "Explicit STS (LSRKStep) solver command line options:" << endl; + cout << " --lsrkmethod : LSRK method choice" << endl; } // Print user options @@ -545,39 +651,57 @@ void UserOptions::print() cout << " --------------------------------- " << endl; cout << " rtol = " << rtol << endl; cout << " atol = " << atol << endl; - cout << " hfixed = " << hfixed << endl; - cout << " order = " << order << endl; cout << " controller = " << controller << endl; - cout << " max steps = " << maxsteps << endl; - cout << " linear RHS = " << linear << endl; + cout << " hfixed = " << hfixed << endl; cout << " --------------------------------- " << endl; - cout << endl; - if (ls == "sludist") + if (implicit) { - cout << " Linear solver options:" << endl; + cout << " ARKStep options:" << endl; cout << " --------------------------------- " << endl; + cout << " order = " << order << endl; + cout << " max steps = " << maxsteps << endl; + cout << " linear RHS = " << linear << endl; + cout << " --------------------------------- " << endl; + cout << endl; + if (ls == "sludist") + { + cout << " Linear solver options:" << endl; + cout << " --------------------------------- " << endl; #if defined(HAVE_HIP) - cout << " LS = SuperLU_DIST (HIP enabled)" << endl; + cout << " LS = SuperLU_DIST (HIP enabled)" << endl; #elif defined(HAVE_CUDA) - cout << " LS = SuperLU_DIST (CUDA enabled)" << endl; + cout << " LS = SuperLU_DIST (CUDA enabled)" << endl; #else - cout << " LS = SuperLU_DIST" << endl; + cout << " LS = SuperLU_DIST" << endl; #endif - cout << " LS info = " << lsinfo << endl; - cout << " msbp = " << msbp << endl; - cout << " --------------------------------- " << endl; + cout << " LS info = " << lsinfo << endl; + cout << " msbp = " << msbp << endl; + cout << " --------------------------------- " << endl; + } + else + { + cout << " Linear solver options:" << endl; + cout << " --------------------------------- " << endl; + cout << " LS = " << ls << endl; + cout << " precond = " << preconditioning << endl; + cout << " LS info = " << lsinfo << endl; + cout << " LS iters = " << liniters << endl; + cout << " msbp = " << msbp << endl; + cout << " epslin = " << epslin << endl; + cout << " --------------------------------- " << endl; + } } else { - cout << " Linear solver options:" << endl; + cout << " LSRKStep options:" << endl; cout << " --------------------------------- " << endl; - cout << " LS = " << ls << endl; - cout << " precond = " << preconditioning << endl; - cout << " LS info = " << lsinfo << endl; - cout << " LS iters = " << liniters << endl; - cout << " msbp = " << msbp << endl; - cout << " epslin = " << epslin << endl; + switch (lsrkmethod) + { + case (ARKODE_LSRK_RKC_2): cout << " method = RKC_2 " << endl; break; + case (ARKODE_LSRK_RKL_2): cout << " method = RKL_2 " << endl; break; + default: cout << " ERROR: illegal lsrkmethod " << endl; + } cout << " --------------------------------- " << endl; } } diff --git a/benchmarks/diffusion_2D/main_cvode.cpp b/benchmarks/diffusion_2D/main_cvode.cpp index 71172e4ce9..47420cdc98 100644 --- a/benchmarks/diffusion_2D/main_cvode.cpp +++ b/benchmarks/diffusion_2D/main_cvode.cpp @@ -310,7 +310,7 @@ int main(int argc, char* argv[]) sunrealtype dTout = udata.tf / uout.nout; sunrealtype tout = dTout; - // Inital output + // Initial output flag = uout.open(&udata); if (check_flag(&flag, "UserOutput::open", 1)) { return 1; } @@ -479,7 +479,7 @@ void UserOptions::help() cout << endl; cout << "Integrator command line options:" << endl; cout << " --rtol : relative tolerance" << endl; - cout << " --atol : absoltue tolerance" << endl; + cout << " --atol : absolute tolerance" << endl; cout << " --ls : linear solver" << endl; cout << " --lsinfo : output residual history" << endl; cout << " --liniters : max number of iterations" << endl; diff --git a/benchmarks/diffusion_2D/main_ida.cpp b/benchmarks/diffusion_2D/main_ida.cpp index 515256ddd9..ccd86c7ad5 100644 --- a/benchmarks/diffusion_2D/main_ida.cpp +++ b/benchmarks/diffusion_2D/main_ida.cpp @@ -305,7 +305,7 @@ int main(int argc, char* argv[]) sunrealtype dTout = udata.tf / uout.nout; sunrealtype tout = dTout; - // Inital output + // Initial output flag = uout.open(&udata); if (check_flag(&flag, "UserOutput::open", 1)) { return 1; } @@ -460,7 +460,7 @@ void UserOptions::help() cout << endl; cout << "Integrator command line options:" << endl; cout << " --rtol : relative tolerance" << endl; - cout << " --atol : absoltue tolerance" << endl; + cout << " --atol : absolute tolerance" << endl; cout << " --ls : linear solver" << endl; cout << " --liniters : max number of iterations" << endl; cout << " --epslin : linear tolerance factor" << endl; diff --git a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt index beb5d1a439..bedab66e97 100644 --- a/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_gpu/CMakeLists.txt @@ -13,7 +13,7 @@ # ------------------------------------------------------------------------------ # list of tests -set(tests ) +set(tests) if(BUILD_ARKODE) if(BUILD_NVECTOR_CUDA) @@ -50,13 +50,8 @@ foreach(test_tuple ${tests}) list(GET test_tuple 1 problem_type) list(GET test_tuple 2 backend) - set(sources - ${benchmark_prefix}/main_${package}.cpp - ${shared_sources} - buffers.cpp - diffusion.cpp - solution.cpp - utils.cpp) + set(sources ${benchmark_prefix}/main_${package}.cpp ${shared_sources} + buffers.cpp diffusion.cpp solution.cpp utils.cpp) if("${backend}" STREQUAL "USE_CUDA") @@ -74,18 +69,13 @@ foreach(test_tuple ${tests}) add_executable(${target} ${sources}) - # if("${backend}" STREQUAL "USE_CUDA") - # sundials_add_benchmark(${target} ${target} diffusion_2D - # ENABLE_GPU - # NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} - # ) - #endif() + # if("${backend}" STREQUAL "USE_CUDA") sundials_add_benchmark(${target} + # ${target} diffusion_2D ENABLE_GPU NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} ) + # endif() if("${backend}" STREQUAL "USE_HIP") - sundials_add_benchmark(${target} ${target} diffusion_2D - ENABLE_GPU - NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS} - ) + sundials_add_benchmark(${target} ${target} diffusion_2D ENABLE_GPU + NUM_CORES ${SUNDIALS_BENCHMARK_NUM_GPUS}) endif() @@ -99,11 +89,9 @@ foreach(test_tuple ${tests}) target_include_directories(${target} PRIVATE ${benchmark_prefix}) - target_link_libraries(${target} - PRIVATE - sundials_${package} - sundials_nvecmpiplusx - sundials_nveccuda) + target_link_libraries( + ${target} PRIVATE sundials_${package} sundials_nvecmpiplusx + sundials_nveccuda) else() @@ -111,12 +99,9 @@ foreach(test_tuple ${tests}) target_include_directories(${target} PRIVATE ${benchmark_prefix}) - target_link_libraries(${target} - PRIVATE - sundials_${package} - sundials_nvecmpiplusx - sundials_nvechip - hip::device) + target_link_libraries( + ${target} PRIVATE sundials_${package} sundials_nvecmpiplusx + sundials_nvechip hip::device) endif() @@ -125,6 +110,6 @@ foreach(test_tuple ${tests}) target_link_libraries(${target} PRIVATE ${MPI_CXX_LIBRARIES}) install(TARGETS ${target} - DESTINATION "${BENCHMARKS_INSTALL_PATH}/diffusion_2D") + DESTINATION "${BENCHMARKS_INSTALL_PATH}/diffusion_2D") endforeach() diff --git a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt index 5e1e5c8862..a5cccdf1ed 100644 --- a/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt +++ b/benchmarks/diffusion_2D/mpi_serial/CMakeLists.txt @@ -13,7 +13,7 @@ # ------------------------------------------------------------------------------ # list of tests -set(tests ) +set(tests) if(BUILD_ARKODE) list(APPEND tests "arkode\;BENCHMARK_ODE") @@ -34,13 +34,8 @@ foreach(test_tuple ${tests}) list(GET test_tuple 0 package) list(GET test_tuple 1 problem_type) - set(sources - ${benchmark_prefix}/main_${package}.cpp - ${shared_sources} - buffers.cpp - diffusion.cpp - solution.cpp - utils.cpp) + set(sources ${benchmark_prefix}/main_${package}.cpp ${shared_sources} + buffers.cpp diffusion.cpp solution.cpp utils.cpp) # set the target name set(target ${package}_diffusion_2D_mpi) @@ -56,25 +51,19 @@ foreach(test_tuple ${tests}) target_include_directories(${target} PRIVATE ${benchmark_prefix}) - target_link_libraries(${target} - PRIVATE - sundials_${package} - sundials_nvecparallel - MPI::MPI_CXX) + target_link_libraries(${target} PRIVATE sundials_${package} + sundials_nvecparallel MPI::MPI_CXX) if(BUILD_SUNLINSOL_SUPERLUDIST) target_compile_definitions(${target} PRIVATE USE_SUPERLU_DIST) - target_link_libraries(${target} - PRIVATE - sundials_sunlinsolsuperludist - sundials_sunmatrixslunrloc) + target_link_libraries(${target} PRIVATE sundials_sunlinsolsuperludist + sundials_sunmatrixslunrloc) endif() install(TARGETS ${target} - DESTINATION "${BENCHMARKS_INSTALL_PATH}/diffusion_2D") + DESTINATION "${BENCHMARKS_INSTALL_PATH}/diffusion_2D") sundials_add_benchmark(${target} ${target} diffusion_2D - NUM_CORES ${SUNDIALS_BENCHMARK_NUM_CPUS} - ) + NUM_CORES ${SUNDIALS_BENCHMARK_NUM_CPUS}) endforeach() diff --git a/benchmarks/nvector/cuda/CMakeLists.txt b/benchmarks/nvector/cuda/CMakeLists.txt index d4db7b8c4d..2a6741fc40 100644 --- a/benchmarks/nvector/cuda/CMakeLists.txt +++ b/benchmarks/nvector/cuda/CMakeLists.txt @@ -14,8 +14,8 @@ message(STATUS "Added CUDA NVECTOR benchmark") -sundials_add_nvector_benchmark(nvector_cuda_benchmark +sundials_add_nvector_benchmark( + nvector_cuda_benchmark SOURCES test_nvector_performance_cuda.cu SUNDIALS_TARGETS sundials_nveccuda - INSTALL_SUBDIR nvector/cuda - ) + INSTALL_SUBDIR nvector/cuda) diff --git a/benchmarks/nvector/hip/CMakeLists.txt b/benchmarks/nvector/hip/CMakeLists.txt index f3b79304d1..f19d4df759 100644 --- a/benchmarks/nvector/hip/CMakeLists.txt +++ b/benchmarks/nvector/hip/CMakeLists.txt @@ -14,8 +14,8 @@ message(STATUS "Added HIP NVECTOR benchmark") -sundials_add_nvector_benchmark(nvector_hip_benchmark +sundials_add_nvector_benchmark( + nvector_hip_benchmark SOURCES test_nvector_performance_hip.cpp SUNDIALS_TARGETS sundials_nvechip - INSTALL_SUBDIR nvector/hip - ) + INSTALL_SUBDIR nvector/hip) diff --git a/benchmarks/nvector/kokkos/CMakeLists.txt b/benchmarks/nvector/kokkos/CMakeLists.txt index 43e7177166..2ca1539df0 100644 --- a/benchmarks/nvector/kokkos/CMakeLists.txt +++ b/benchmarks/nvector/kokkos/CMakeLists.txt @@ -14,15 +14,16 @@ message(STATUS "Added Kokkos NVECTOR benchmark") foreach(backend ${KOKKOS_EXAMPLES_BACKENDS}) - sundials_add_nvector_benchmark(test_nvector_performance_kokkos.${backend} + sundials_add_nvector_benchmark( + test_nvector_performance_kokkos.${backend} SOURCES test_nvector_performance_kokkos.cpp SUNDIALS_TARGETS sundials_core sundials_nveckokkos - INSTALL_SUBDIR nvector/kokkos - ) + INSTALL_SUBDIR nvector/kokkos) - target_compile_definitions(test_nvector_performance_kokkos.${backend} PRIVATE USE_${backend}) + target_compile_definitions(test_nvector_performance_kokkos.${backend} + PRIVATE USE_${backend}) install(TARGETS test_nvector_performance_kokkos.${backend} - DESTINATION "${BENCHMARKS_INSTALL_PATH}/") + DESTINATION "${BENCHMARKS_INSTALL_PATH}/") endforeach() diff --git a/benchmarks/nvector/mpiplusx/CMakeLists.txt b/benchmarks/nvector/mpiplusx/CMakeLists.txt index ec6c3c49d9..6a1fd9c1f5 100644 --- a/benchmarks/nvector/mpiplusx/CMakeLists.txt +++ b/benchmarks/nvector/mpiplusx/CMakeLists.txt @@ -14,9 +14,9 @@ message(STATUS "Added MPIPlusX NVECTOR benchmark") -sundials_add_nvector_benchmark(nvector_mpiplusx_benchmark +sundials_add_nvector_benchmark( + nvector_mpiplusx_benchmark SOURCES test_nvector_performance_mpiplusx.c SUNDIALS_TARGETS sundials_nvecserial sundials_nvecmpiplusx LINK_LIBRARIES MPI::MPI_CXX - INSTALL_SUBDIR nvector/mpiplusx - ) \ No newline at end of file + INSTALL_SUBDIR nvector/mpiplusx) diff --git a/benchmarks/nvector/openmp/CMakeLists.txt b/benchmarks/nvector/openmp/CMakeLists.txt index 96c9f9e3b0..0980bb8ef4 100644 --- a/benchmarks/nvector/openmp/CMakeLists.txt +++ b/benchmarks/nvector/openmp/CMakeLists.txt @@ -16,14 +16,13 @@ message(STATUS "Added OpenMP NVECTOR benchmark") -# Set-up linker flags and link libraries -# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") -# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") -# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_C_FLAGS}") +# Set-up linker flags and link libraries set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} +# ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} +# ${OpenMP_CXX_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} +# ${OpenMP_C_FLAGS}") -sundials_add_nvector_benchmark(nvector_openmp_benchmark +sundials_add_nvector_benchmark( + nvector_openmp_benchmark SOURCES test_nvector_performance_openmp.c SUNDIALS_TARGETS sundials_nvecopenmp - INSTALL_SUBDIR nvector/openmp - ) - + INSTALL_SUBDIR nvector/openmp) diff --git a/benchmarks/nvector/openmpdev/CMakeLists.txt b/benchmarks/nvector/openmpdev/CMakeLists.txt index c78d4821d6..54789c9a95 100644 --- a/benchmarks/nvector/openmpdev/CMakeLists.txt +++ b/benchmarks/nvector/openmpdev/CMakeLists.txt @@ -22,13 +22,14 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_C_FLAGS}") -add_executable(test_nvector_performance_openmpdev - test_nvector_performance_openmpdev.c - ../test_nvector_performance.c +add_executable( + test_nvector_performance_openmpdev + test_nvector_performance_openmpdev.c ../test_nvector_performance.c ../../../src/sundials/sundials_nvector.c) # folder to organize targets in an IDE -set_target_properties(test_nvector_performance_openmp PROPERTIES FOLDER "Benchmarks") +set_target_properties(test_nvector_performance_openmp PROPERTIES FOLDER + "Benchmarks") target_include_directories(test_nvector_performance_openmpdev PRIVATE ..) @@ -36,4 +37,4 @@ target_include_directories(test_nvector_performance_openmpdev PRIVATE ..) target_link_libraries(test_nvector_performance_openmpdev ${SUNDIALS_LIBS}) install(TARGETS test_nvector_performance_openmpdev - DESTINATION "${BENCHMARKS_INSTALL_PATH}/nvector/openmpdev") + DESTINATION "${BENCHMARKS_INSTALL_PATH}/nvector/openmpdev") diff --git a/benchmarks/nvector/parallel/CMakeLists.txt b/benchmarks/nvector/parallel/CMakeLists.txt index c096594f3d..87ed3138ba 100644 --- a/benchmarks/nvector/parallel/CMakeLists.txt +++ b/benchmarks/nvector/parallel/CMakeLists.txt @@ -14,9 +14,9 @@ message(STATUS "Added Parallel NVECTOR benchmark") -sundials_add_nvector_benchmark(nvector_parallel_benchmark +sundials_add_nvector_benchmark( + nvector_parallel_benchmark SOURCES test_nvector_performance_parallel.c SUNDIALS_TARGETS sundials_nvecparallel LINK_LIBRARIES MPI::MPI_CXX - INSTALL_SUBDIR nvector/parallel - ) + INSTALL_SUBDIR nvector/parallel) diff --git a/benchmarks/nvector/parhyp/CMakeLists.txt b/benchmarks/nvector/parhyp/CMakeLists.txt index 7e05ff3a0f..216085064b 100644 --- a/benchmarks/nvector/parhyp/CMakeLists.txt +++ b/benchmarks/nvector/parhyp/CMakeLists.txt @@ -16,9 +16,9 @@ message(STATUS "Added hypre NVECTOR benchmark") -sundials_add_nvector_benchmark(nvector_parhyp_benchmark +sundials_add_nvector_benchmark( + nvector_parhyp_benchmark SOURCES test_nvector_performance_parhyp.c SUNDIALS_TARGETS sundials_nvecparhyp LINK_LIBRARIES MPI::MPI_C SUNDIALS::HYPRE - INSTALL_SUBDIR nvector/parhyp - ) + INSTALL_SUBDIR nvector/parhyp) diff --git a/benchmarks/nvector/petsc/CMakeLists.txt b/benchmarks/nvector/petsc/CMakeLists.txt index 63e2f8a1e9..fd9183593d 100644 --- a/benchmarks/nvector/petsc/CMakeLists.txt +++ b/benchmarks/nvector/petsc/CMakeLists.txt @@ -16,9 +16,9 @@ message(STATUS "Added PETSc NVECTOR benchmark") -sundials_add_nvector_benchmark(nvector_petsc_benchmark +sundials_add_nvector_benchmark( + nvector_petsc_benchmark SOURCES test_nvector_performance_petsc.c SUNDIALS_TARGETS sundials_nvecpetsc LINK_LIBRARIES MPI::MPI_C PUBLIC SUNDIALS::PETSC - INSTALL_SUBDIR nvector/petsc - ) + INSTALL_SUBDIR nvector/petsc) diff --git a/benchmarks/nvector/plot_nvector_performance_results.py b/benchmarks/nvector/plot_nvector_performance_results.py index 02c45665e6..c5e9f69301 100755 --- a/benchmarks/nvector/plot_nvector_performance_results.py +++ b/benchmarks/nvector/plot_nvector_performance_results.py @@ -15,11 +15,12 @@ # This script plots the output from test_nector_performance_* and assumes: # 1. vector lengths are powers of two starting from 0, and # 2. output files are named: output_nelem_nvec_nsum_ntest_timing.txt -# where nelem is the number of elements in the vector, nvec is the nuber of +# where nelem is the number of elements in the vector, nvec is the number of # vectors, nsum is the number of sums, ntest is the number of tests, and timing # indicates if timing was enabled. # ----------------------------------------------------------------------------- + def main(): import argparse @@ -35,42 +36,60 @@ def main(): import matplotlib.ticker as mtick parser = argparse.ArgumentParser( - description='Plot data from NVector performance tests') - - parser.add_argument('op', type=str, - help='Which NVector operation to plot') - - parser.add_argument('datadir', type=str, - help='Directory where test output files are located') - - parser.add_argument('--timevelem', dest='timevelem', action='store_true', - help='Turn on plots for time vs number of elements') - - parser.add_argument('--noheatmap', dest='heatmap', action='store_false', - help='Turn off heatmap plots') - - parser.add_argument('--loglog', dest='loglog', action='store_true', - help='Generate loglog plots for time vs number of elements') - - parser.add_argument('--show', dest='show', action='store_true', - help='Display plots rather than saving to file') - - parser.add_argument('--debug', dest='debug', action='store_true', - help='Turn on debugging output') + description="Plot data from NVector performance tests" + ) + + parser.add_argument("op", type=str, help="Which NVector operation to plot") + + parser.add_argument( + "datadir", type=str, help="Directory where test output files are located" + ) + + parser.add_argument( + "--timevelem", + dest="timevelem", + action="store_true", + help="Turn on plots for time vs number of elements", + ) + + parser.add_argument( + "--noheatmap", + dest="heatmap", + action="store_false", + help="Turn off heatmap plots", + ) + + parser.add_argument( + "--loglog", + dest="loglog", + action="store_true", + help="Generate loglog plots for time vs number of elements", + ) + + parser.add_argument( + "--show", + dest="show", + action="store_true", + help="Display plots rather than saving to file", + ) + + parser.add_argument( + "--debug", dest="debug", action="store_true", help="Turn on debugging output" + ) # parse command line args args = parser.parse_args() - if (args.debug): + if args.debug: print(args) # check for test data directory - if (not os.path.isdir(args.datadir)): - print("ERROR:",args.datadir,"does not exist") + if not os.path.isdir(args.datadir): + print("ERROR:", args.datadir, "does not exist") sys.exit() # sort output files - output = sorted(glob.glob(args.datadir+'/output*.txt')) + output = sorted(glob.glob(args.datadir + "/output*.txt")) # if (args.debug): # print("output files") @@ -80,8 +99,8 @@ def main(): # figure out vector sizes, number of vectors, and number of sums nelem = [] - nvec = [] - nsum = [] + nvec = [] + nsum = [] ntest = [] # parse file names to get input parameters @@ -95,32 +114,32 @@ def main(): ns = int(split_fout[3]) nt = int(split_fout[4]) - if (not ne in nelem): + if not ne in nelem: nelem.append(ne) - if (not nv in nvec): + if not nv in nvec: nvec.append(nv) - if (not ns in nsum): + if not ns in nsum: nsum.append(ns) - if (not nt in ntest): + if not nt in ntest: ntest.append(nt) - if (len(ntest) != 1): + if len(ntest) != 1: print("Warning: Unequal numbers of tests") - if (args.debug): - print("nelem:",nelem, len(nelem)) - print("nvec: ",nvec, len(nvec)) - print("nsum: ",nsum, len(nsum)) - print("ntest:",ntest, len(ntest)) + if args.debug: + print("nelem:", nelem, len(nelem)) + print("nvec: ", nvec, len(nvec)) + print("nsum: ", nsum, len(nsum)) + print("ntest:", ntest, len(ntest)) # allocate numpy arrays for timing data - avg_fused = np.zeros([len(nvec), len(nelem)]) + avg_fused = np.zeros([len(nvec), len(nelem)]) sdev_fused = np.zeros([len(nvec), len(nelem)]) - avg_unfused = np.zeros([len(nvec), len(nelem)]) + avg_unfused = np.zeros([len(nvec), len(nelem)]) sdev_unfused = np.zeros([len(nvec), len(nelem)]) avg_ratio = np.zeros([len(nvec), len(nelem)]) @@ -131,8 +150,8 @@ def main(): # read output files for f in output: - if (args.debug): - print("Reading:",f) + if args.debug: + print("Reading:", f) # get test inputs from file name split_fout = f.split("/")[-1] @@ -149,15 +168,15 @@ def main(): split_line = shlex.split(line) # skip blank lines - if (not split_line): + if not split_line: continue # tests finished, stop reading file - if (split_line[0] == "Finished"): + if split_line[0] == "Finished": break # check if the operation is the one we want and get data - if (args.op == split_line[0]): + if args.op == split_line[0]: i = nvec.index(nv) j = nelem.index(ne) @@ -165,15 +184,15 @@ def main(): # NVEC[i][j] = nv # NELM[i][j] = ne - avg_fused[i][j] = float(split_line[1]) + avg_fused[i][j] = float(split_line[1]) sdev_fused[i][j] = float(split_line[2]) - avg_unfused[i][j] = float(split_line[5]) + avg_unfused[i][j] = float(split_line[5]) sdev_unfused[i][j] = float(split_line[6]) avg_ratio[i][j] = avg_fused[i][j] / avg_unfused[i][j] - if (args.debug): + if args.debug: print(avg_fused) print(avg_unfused) print(avg_ratio) @@ -185,35 +204,37 @@ def main(): # -------------------------------------------------------------------------- # allocate arrays for the upper and lower bounds of the confidence interval - lower_fused = np.zeros([len(nvec), len(nelem)]) - upper_fused = np.zeros([len(nvec), len(nelem)]) + lower_fused = np.zeros([len(nvec), len(nelem)]) + upper_fused = np.zeros([len(nvec), len(nelem)]) lower_unfused = np.zeros([len(nvec), len(nelem)]) upper_unfused = np.zeros([len(nvec), len(nelem)]) # critical value for 99% confidence interval - if (ntest[0] < 30): + if ntest[0] < 30: # student's t distribution - cv = st.t.interval(0.99, ntest[0]-1)[1] + cv = st.t.interval(0.99, ntest[0] - 1)[1] else: # normal distribution cv = st.norm.ppf(0.995) # confidence intervals - cdev_fused = cv * sdev_fused / np.sqrt(ntest[0]) + cdev_fused = cv * sdev_fused / np.sqrt(ntest[0]) lower_fused = avg_fused - cdev_fused upper_fused = avg_fused + cdev_fused - cdev_unfused = cv * sdev_unfused / np.sqrt(ntest[0]) + cdev_unfused = cv * sdev_unfused / np.sqrt(ntest[0]) lower_unfused = avg_unfused - cdev_unfused upper_unfused = avg_unfused + cdev_unfused # check if the fused average times are within the unfused confidence interval - fused_in = np.where(np.logical_and(avg_fused < upper_unfused, - avg_fused > lower_unfused)) + fused_in = np.where( + np.logical_and(avg_fused < upper_unfused, avg_fused > lower_unfused) + ) # check if the unfused average times are within the fused confidence interval - unfused_in = np.where(np.logical_and(avg_unfused < upper_fused, - avg_unfused > lower_fused)) + unfused_in = np.where( + np.logical_and(avg_unfused < upper_fused, avg_unfused > lower_fused) + ) # get which numbers of vectors and elements for fused tests are in the # confidence interval of the unfused times @@ -226,7 +247,7 @@ def main(): ef[i] = np.log2(nelem[fused_in[1][i]]) df[i] = 1 - if (args.debug): + if args.debug: print(vf) print(ef) @@ -241,7 +262,7 @@ def main(): eu[i] = np.log2(nelem[unfused_in[1][i]]) du[i] = 1 - if (args.debug): + if args.debug: print(vu) print(eu) @@ -266,20 +287,20 @@ def main(): # print(NELM) # print(avg_ratio) for i in reversed(range(len(nvec))): - print('%2d' % int(i+1), str(avg_ratio[i]).replace('\n', '')) + print("%2d" % int(i + 1), str(avg_ratio[i]).replace("\n", "")) print # -------------------------------------------------------------------------- # Heat Map # -------------------------------------------------------------------------- - if (args.heatmap): + if args.heatmap: - x = np.arange(len(nelem)+1)-0.5 # x = log2(number of elements) = 0,1,2,... - y = np.arange(len(nvec)+1)+1.5 # y = number of vectors = 2,3,4,... + x = np.arange(len(nelem) + 1) - 0.5 # x = log2(number of elements) = 0,1,2,... + y = np.arange(len(nvec) + 1) + 1.5 # y = number of vectors = 2,3,4,... # y = np.arange(len(nvec)+1)+0.5 # y = number of vectors = 1,2,3,... X, Y = np.meshgrid(x, y) - if (args.debug): + if args.debug: print(x) print(y) @@ -287,67 +308,79 @@ def main(): rmax = np.amax(avg_ratio) rmin = np.amin(avg_ratio) - ext = 'neither' - if (rmin > 1): - cmap='Reds' - norm = mpl.colors.Normalize(vmin=rmin, vmax=min(rmax,2)) - v = np.linspace(rmin, min(rmax,2), 10, endpoint=True) - if (rmax > 2): - ext = 'max' + ext = "neither" + if rmin > 1: + cmap = "Reds" + norm = mpl.colors.Normalize(vmin=rmin, vmax=min(rmax, 2)) + v = np.linspace(rmin, min(rmax, 2), 10, endpoint=True) + if rmax > 2: + ext = "max" else: - cmap='seismic' - if (rmax-1 > 1): + cmap = "seismic" + if rmax - 1 > 1: rrange = 1 - ext = 'max' + ext = "max" else: - rrange = max(abs(rmax-1),abs(rmin-1)) + rrange = max(abs(rmax - 1), abs(rmin - 1)) - v1 = np.linspace(1-rrange, 1, 5, endpoint=True) - v2 = np.linspace(1, 1+rrange, 5, endpoint=True) - v = np.append(v1,v2[1:]) - norm = mpl.colors.Normalize(vmin=1-rrange, vmax=1+rrange) + v1 = np.linspace(1 - rrange, 1, 5, endpoint=True) + v2 = np.linspace(1, 1 + rrange, 5, endpoint=True) + v = np.append(v1, v2[1:]) + norm = mpl.colors.Normalize(vmin=1 - rrange, vmax=1 + rrange) # plot heatmap plt.pcolormesh(X, Y, avg_ratio, cmap=cmap, norm=norm) clb = plt.colorbar(ticks=v, extend=ext) - clb.ax.set_title('Max = {0:.2f}\nMin = {1:.2f}'.format(rmax,rmin)) + clb.ax.set_title("Max = {0:.2f}\nMin = {1:.2f}".format(rmax, rmin)) # aff markers to indicate if the average time falls in a confidence interval - plt.scatter(ef,vf,s=40,marker='^',c=df,label='fused') - plt.scatter(eu,vu,s=40,marker='v',c=du,label='unfused') + plt.scatter(ef, vf, s=40, marker="^", c=df, label="fused") + plt.scatter(eu, vu, s=40, marker="v", c=du, label="unfused") plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1), ncol=2) # add legend for scatter plot art = [] - lgd = plt.legend(loc='lower right', bbox_to_anchor=(1.34, -0.17)) + lgd = plt.legend(loc="lower right", bbox_to_anchor=(1.34, -0.17)) art.append(lgd) # add labels and title plt.xticks(np.log2(nelem)) plt.yticks(nvec) - plt.xlabel('log2(num elements)') - plt.ylabel('num vectors') - plt.title('avg fused time / avg unfused time \n'+args.op) + plt.xlabel("log2(num elements)") + plt.ylabel("num vectors") + plt.title("avg fused time / avg unfused time \n" + args.op) # display or save figure - if (args.show): + if args.show: plt.show() else: - plt.savefig(args.op+'-heatmap.pdf', - additional_artists=art, - bbox_inches="tight") + plt.savefig( + args.op + "-heatmap.pdf", additional_artists=art, bbox_inches="tight" + ) plt.close() # -------------------------------------------------------------------------- # Time vs Number of Elements Plots # -------------------------------------------------------------------------- - if (args.timevelem): - - colors = ['#000000','#a6cee3','#1f78b4','#b2df8a','#33a02c', - '#fb9a99','#e31a1c','#fdbf6f','#ff7f00','#cab2d6', - '#6a3d9a','#ffff99','#b15928'] - - hatch = [ '/','\\','-','+','x','o','O','.','*'] + if args.timevelem: + + colors = [ + "#000000", + "#a6cee3", + "#1f78b4", + "#b2df8a", + "#33a02c", + "#fb9a99", + "#e31a1c", + "#fdbf6f", + "#ff7f00", + "#cab2d6", + "#6a3d9a", + "#ffff99", + "#b15928", + ] + + hatch = ["/", "\\", "-", "+", "x", "o", "O", ".", "*"] # -------------------------------------------------------------------------- # Combined Number of Vectors Plots @@ -359,38 +392,45 @@ def main(): i = nvec.index(nv) - if (args.loglog): - ax.loglog(nelem, avg_fused[i], - color=colors[i], linestyle='-', label=nv) - ax.loglog(nelem, avg_unfused[i], - color=colors[i], linestyle='--', label=None) + if args.loglog: + ax.loglog(nelem, avg_fused[i], color=colors[i], linestyle="-", label=nv) + ax.loglog( + nelem, avg_unfused[i], color=colors[i], linestyle="--", label=None + ) else: - ax.plot(nelem, avg_fused[i], - color=colors[i], linestyle='-', label=nv) - ax.plot(nelem, avg_unfused[i], - color=colors[i], linestyle='--', label=None) + ax.plot(nelem, avg_fused[i], color=colors[i], linestyle="-", label=nv) + ax.plot( + nelem, avg_unfused[i], color=colors[i], linestyle="--", label=None + ) # plot confidence interval - ax.fill_between(nelem, lower_fused[i], upper_fused[i], - color=colors[i], alpha=0.3) - ax.fill_between(nelem, lower_unfused[i], upper_unfused[i], - color=colors[i], hatch='.', alpha=0.3) + ax.fill_between( + nelem, lower_fused[i], upper_fused[i], color=colors[i], alpha=0.3 + ) + ax.fill_between( + nelem, + lower_unfused[i], + upper_unfused[i], + color=colors[i], + hatch=".", + alpha=0.3, + ) ax.legend() ax.grid() - plt.title('Average Time Fused vs Unfused \n'+args.op) - plt.xlabel('vector length') - plt.ylabel('time (s)') + plt.title("Average Time Fused vs Unfused \n" + args.op) + plt.xlabel("vector length") + plt.ylabel("time (s)") - if (args.show): + if args.show: plt.show() else: - if (args.loglog): - fname=args.op+'-nvec-all-loglog.pdf' + if args.loglog: + fname = args.op + "-nvec-all-loglog.pdf" else: - fname=args.op+'-nvec-all.pdf' - plt.ticklabel_format(axis='both',style='sci') + fname = args.op + "-nvec-all.pdf" + plt.ticklabel_format(axis="both", style="sci") plt.savefig(fname) plt.close() @@ -400,49 +440,70 @@ def main(): for nv in nvec: fig = plt.figure() - ax = fig.add_subplot(111) + ax = fig.add_subplot(111) idx = nvec.index(nv) # plot run times - if (args.loglog): - ax.loglog(nelem, avg_fused[idx], - color='red', linestyle='-', label='Fused') - ax.loglog(nelem, avg_unfused[idx], - color='blue', linestyle='--', label='Unfused') + if args.loglog: + ax.loglog( + nelem, avg_fused[idx], color="red", linestyle="-", label="Fused" + ) + ax.loglog( + nelem, + avg_unfused[idx], + color="blue", + linestyle="--", + label="Unfused", + ) else: - ax.plot(nelem, avg_fused[idx], - color='red', linestyle='-', label='Fused') - ax.plot(nelem, avg_unfused[idx], - color='blue', linestyle='--', label='Unfused') + ax.plot( + nelem, avg_fused[idx], color="red", linestyle="-", label="Fused" + ) + ax.plot( + nelem, + avg_unfused[idx], + color="blue", + linestyle="--", + label="Unfused", + ) # plot confidence intervals - ax.fill_between(nelem, lower_fused[idx], upper_fused[idx], - color='red', alpha=0.2) - ax.fill_between(nelem, lower_unfused[idx], upper_unfused[idx], - color='blue', hatch='.', alpha=0.2) + ax.fill_between( + nelem, lower_fused[idx], upper_fused[idx], color="red", alpha=0.2 + ) + ax.fill_between( + nelem, + lower_unfused[idx], + upper_unfused[idx], + color="blue", + hatch=".", + alpha=0.2, + ) ax.legend() ax.grid() - plt.title('Average Time Fused vs Unfused with '+str(nv)+' vectors\n'+args.op) - plt.xlabel('vector length') - ax.set_ylabel('time (s)') + plt.title( + "Average Time Fused vs Unfused with " + str(nv) + " vectors\n" + args.op + ) + plt.xlabel("vector length") + ax.set_ylabel("time (s)") - if (args.show): + if args.show: plt.show() else: - if (args.loglog): - fname=args.op+'-nvec-'+str(nv)+'-loglog.pdf' + if args.loglog: + fname = args.op + "-nvec-" + str(nv) + "-loglog.pdf" else: - fname=args.op+'-nvec-'+str(nv)+'.pdf' - plt.ticklabel_format(axis='both',style='sci') + fname = args.op + "-nvec-" + str(nv) + ".pdf" + plt.ticklabel_format(axis="both", style="sci") plt.savefig(fname) plt.close() + # =============================================================================== if __name__ == "__main__": main() # EOF - diff --git a/benchmarks/nvector/plot_nvector_performance_speedup.py b/benchmarks/nvector/plot_nvector_performance_speedup.py index fb421f5573..baaaf43682 100755 --- a/benchmarks/nvector/plot_nvector_performance_speedup.py +++ b/benchmarks/nvector/plot_nvector_performance_speedup.py @@ -15,11 +15,12 @@ # This script plots the output from test_nector_performance_* and assumes: # 1. vector lengths are powers of two starting from 0, and # 2. output files are named: output_nelem_nvec_nsum_ntest_timing.txt -# where nelem is the number of elements in the vector, nvec is the nuber of +# where nelem is the number of elements in the vector, nvec is the number of # vectors, nsum is the number of sums, ntest is the number of tests, and timing # indicates if timing was enabled. # ----------------------------------------------------------------------------- + def main(): import argparse @@ -35,46 +36,61 @@ def main(): import matplotlib.ticker as mtick parser = argparse.ArgumentParser( - description='Plot data from NVector performance tests') - - parser.add_argument('op', type=str, - help='Which NVector operation to plot') - - parser.add_argument('datadir', type=str, - help='Directory where test output files are located') - - parser.add_argument('--noplots', dest='noplots', action='store_true', - help='Turn on plots for time vs number of elements') - - parser.add_argument('--logx', dest='logx', action='store_true', - help='Generate plots for speedup with log scale for the x axis (number of elements') - - parser.add_argument('--fused', dest='fused', action='store_true', - help='Operation is a fused op') - - parser.add_argument('--show', dest='show', action='store_true', - help='Display plots rather than saving to file') - - parser.add_argument('--debug', dest='debug', action='store_true', - help='Turn on debugging output') + description="Plot data from NVector performance tests" + ) + + parser.add_argument("op", type=str, help="Which NVector operation to plot") + + parser.add_argument( + "datadir", type=str, help="Directory where test output files are located" + ) + + parser.add_argument( + "--noplots", + dest="noplots", + action="store_true", + help="Turn on plots for time vs number of elements", + ) + + parser.add_argument( + "--logx", + dest="logx", + action="store_true", + help="Generate plots for speedup with log scale for the x axis (number of elements", + ) + + parser.add_argument( + "--fused", dest="fused", action="store_true", help="Operation is a fused op" + ) + + parser.add_argument( + "--show", + dest="show", + action="store_true", + help="Display plots rather than saving to file", + ) + + parser.add_argument( + "--debug", dest="debug", action="store_true", help="Turn on debugging output" + ) # parse command line args args = parser.parse_args() - if (args.debug): + if args.debug: print(args) # check for test data directory - if (not os.path.isdir(args.datadir)): - print("ERROR:",args.datadir,"does not exist") + if not os.path.isdir(args.datadir): + print("ERROR:", args.datadir, "does not exist") sys.exit() # sort output files - output_baseline = sorted(glob.glob(args.datadir+'/output*-old.log')) - output_new = sorted(glob.glob(args.datadir+'/output*-new.log')) + output_baseline = sorted(glob.glob(args.datadir + "/output*-old.log")) + output_new = sorted(glob.glob(args.datadir + "/output*-new.log")) output = output_baseline + output_new - if (args.debug): + if args.debug: print("output files") print(len(output)) for i in range(len(output)): @@ -82,8 +98,8 @@ def main(): # figure out vector sizes, number of vectors, and number of sums nelem = [] - nvec = [] - nsum = [] + nvec = [] + nsum = [] ntest = [] # parse file names to get input parameters @@ -97,40 +113,40 @@ def main(): ns = int(split_fout[3]) nt = int(split_fout[4]) - if (not ne in nelem): + if not ne in nelem: nelem.append(ne) - if (not nv in nvec): + if not nv in nvec: nvec.append(nv) - if (not ns in nsum): + if not ns in nsum: nsum.append(ns) - if (not nt in ntest): + if not nt in ntest: ntest.append(nt) - if (len(ntest) != 1): + if len(ntest) != 1: print("Warning: Unequal numbers of tests") nelem.sort() - if (args.debug): - print("nelem:",nelem, len(nelem)) - print("nvec: ",nvec, len(nvec)) - print("nsum: ",nsum, len(nsum)) - print("ntest:",ntest, len(ntest)) + if args.debug: + print("nelem:", nelem, len(nelem)) + print("nvec: ", nvec, len(nvec)) + print("nsum: ", nsum, len(nsum)) + print("ntest:", ntest, len(ntest)) # allocate numpy arrays for timing data - avg_denom = np.zeros([len(nvec), len(nelem)]) + avg_denom = np.zeros([len(nvec), len(nelem)]) sdev_denom = np.zeros([len(nvec), len(nelem)]) - avg_numer = np.zeros([len(nvec), len(nelem)]) + avg_numer = np.zeros([len(nvec), len(nelem)]) sdev_numer = np.zeros([len(nvec), len(nelem)]) avg_ratio = np.zeros([len(nvec), len(nelem)]) # read 'baseline' files for f in output_baseline: - if (args.debug): - print("Reading:",f) + if args.debug: + print("Reading:", f) # get test inputs from file name split_fout = f.split("/")[-1] split_fout = split_fout.split("_") @@ -142,22 +158,22 @@ def main(): # split line into list split_line = shlex.split(line) # skip blank lines - if (not split_line): + if not split_line: continue # tests finished, stop reading file - if (split_line[0] == "Finished"): + if split_line[0] == "Finished": break # check if the operation is the one we want and get data - if (args.op == split_line[0]): + if args.op == split_line[0]: i = nvec.index(nv) j = nelem.index(ne) - avg_numer[i][j] = float(split_line[1]) + avg_numer[i][j] = float(split_line[1]) sdev_numer[i][j] = float(split_line[2]) # read output files for f in output_new: - if (args.debug): - print("Reading:",f) + if args.debug: + print("Reading:", f) # get test inputs from file name split_fout = f.split("/")[-1] split_fout = split_fout.split("_") @@ -169,16 +185,16 @@ def main(): # split line into list split_line = shlex.split(line) # skip blank lines - if (not split_line): + if not split_line: continue # tests finished, stop reading file - if (split_line[0] == "Finished"): + if split_line[0] == "Finished": break # check if the operation is the one we want and get data - if (args.op == split_line[0]): + if args.op == split_line[0]: i = nvec.index(nv) j = nelem.index(ne) - avg_denom[i][j] = float(split_line[1]) + avg_denom[i][j] = float(split_line[1]) sdev_denom[i][j] = float(split_line[2]) avg_ratio[i][j] = avg_numer[i][j] / avg_denom[i][j] @@ -187,35 +203,37 @@ def main(): # -------------------------------------------------------------------------- # allocate arrays for the upper and lower bounds of the confidence interval - lower_denom = np.zeros([len(nvec), len(nelem)]) - upper_denom = np.zeros([len(nvec), len(nelem)]) + lower_denom = np.zeros([len(nvec), len(nelem)]) + upper_denom = np.zeros([len(nvec), len(nelem)]) lower_numer = np.zeros([len(nvec), len(nelem)]) upper_numer = np.zeros([len(nvec), len(nelem)]) # critical value for 99% confidence interval - if (ntest[0] < 30): + if ntest[0] < 30: # student's t distribution - cv = st.t.interval(0.99, ntest[0]-1)[1] + cv = st.t.interval(0.99, ntest[0] - 1)[1] else: # normal distribution cv = st.norm.ppf(0.995) # confidence intervals - cdev_denom = cv * sdev_denom / np.sqrt(ntest[0]) + cdev_denom = cv * sdev_denom / np.sqrt(ntest[0]) lower_denom = avg_denom - cdev_denom upper_denom = avg_denom + cdev_denom - cdev_numer = cv * sdev_numer / np.sqrt(ntest[0]) + cdev_numer = cv * sdev_numer / np.sqrt(ntest[0]) lower_numer = avg_numer - cdev_numer upper_numer = avg_numer + cdev_numer # check if the new average times are within the baseline confidence interval - denom_in = np.where(np.logical_and(avg_denom < upper_numer, - avg_denom > lower_numer)) + denom_in = np.where( + np.logical_and(avg_denom < upper_numer, avg_denom > lower_numer) + ) # check if the baseline average times are within the new confidence interval - numer_in = np.where(np.logical_and(avg_numer < upper_denom, - avg_numer > lower_denom)) + numer_in = np.where( + np.logical_and(avg_numer < upper_denom, avg_numer > lower_denom) + ) # get which numbers of vectors and elements for new tests are in the # confidence interval of the baseline times @@ -228,9 +246,9 @@ def main(): ef[i] = np.log2(nelem[denom_in[1][i]]) df[i] = 1 - if (args.debug): - print('vf:', vf) - print('ef:', ef) + if args.debug: + print("vf:", vf) + print("ef:", ef) # get which numbers of vectors and elements for baseline tests are in the # confidence interval of the new times @@ -243,9 +261,9 @@ def main(): eu[i] = np.log2(nelem[numer_in[1][i]]) du[i] = 1 - if (args.debug): - print('vu:', vu) - print('eu:', eu) + if args.debug: + print("vu:", vu) + print("eu:", eu) # -------------------------------------------------------------------------- # Output ratios @@ -256,29 +274,41 @@ def main(): print("avg. new") for i in reversed(range(len(nvec))): - print('%2d' % int(i+1), str(avg_denom[i]).replace('\n', '')) + print("%2d" % int(i + 1), str(avg_denom[i]).replace("\n", "")) print() print("avg. baseline") for i in reversed(range(len(nvec))): - print('%2d' % int(i+1), str(avg_numer[i]).replace('\n', '')) + print("%2d" % int(i + 1), str(avg_numer[i]).replace("\n", "")) print() print("avg. ratio (speedup)") for i in reversed(range(len(nvec))): - print('%2d' % int(i+1), str(avg_ratio[i]).replace('\n', '')) + print("%2d" % int(i + 1), str(avg_ratio[i]).replace("\n", "")) print() # -------------------------------------------------------------------------- # Speedup v. Number of Elements Plots # -------------------------------------------------------------------------- - if (not args.noplots): - - colors = ['#000000','#a6cee3','#1f78b4','#b2df8a','#33a02c', - '#fb9a99','#e31a1c','#fdbf6f','#ff7f00','#cab2d6', - '#6a3d9a','#ffff99','#b15928'] - - hatch = [ '/','\\','-','+','x','o','O','.','*'] + if not args.noplots: + + colors = [ + "#000000", + "#a6cee3", + "#1f78b4", + "#b2df8a", + "#33a02c", + "#fb9a99", + "#e31a1c", + "#fdbf6f", + "#ff7f00", + "#cab2d6", + "#6a3d9a", + "#ffff99", + "#b15928", + ] + + hatch = ["/", "\\", "-", "+", "x", "o", "O", ".", "*"] # -------------------------------------------------------------------------- # Combined Number of Vectors Plots @@ -287,19 +317,17 @@ def main(): ax = fig.add_subplot(111) if args.fused: - indices = range(0,len(nvec)) + indices = range(0, len(nvec)) else: - indices = range(len(nvec)-1,len(nvec)) + indices = range(len(nvec) - 1, len(nvec)) for i in indices: - lab = 'num. vecs %d' % nvec[i] - if (args.logx): - ax.plot(nelem, avg_ratio[i], - color=colors[i], linestyle='-', label=lab) - ax.set_xscale('log') + lab = "num. vecs %d" % nvec[i] + if args.logx: + ax.plot(nelem, avg_ratio[i], color=colors[i], linestyle="-", label=lab) + ax.set_xscale("log") else: - ax.plot(nelem, avg_ratio[i], - color=colors[i], linestyle='-', label=lab) + ax.plot(nelem, avg_ratio[i], color=colors[i], linestyle="-", label=lab) # # plot confidence interval # ax.fill_between(nelem, lower_denom[i], upper_denom[i], # color=colors[i], alpha=0.3) @@ -309,18 +337,18 @@ def main(): ax.legend() ax.grid() - plt.title('Average Speedup \n'+args.op) - plt.xlabel('vector length') - plt.ylabel('speedup (baseline/new)') + plt.title("Average Speedup \n" + args.op) + plt.xlabel("vector length") + plt.ylabel("speedup (baseline/new)") - if (args.show): + if args.show: plt.show() else: - if (args.logx): - fname=args.op+'-nvec-all-logx.pdf' + if args.logx: + fname = args.op + "-nvec-all-logx.pdf" else: - fname=args.op+'-nvec-all.pdf' - plt.ticklabel_format(axis='both',style='sci') + fname = args.op + "-nvec-all.pdf" + plt.ticklabel_format(axis="both", style="sci") plt.savefig(fname) plt.close() diff --git a/benchmarks/nvector/pthreads/CMakeLists.txt b/benchmarks/nvector/pthreads/CMakeLists.txt index ae7ec038c0..bc7cac35f2 100644 --- a/benchmarks/nvector/pthreads/CMakeLists.txt +++ b/benchmarks/nvector/pthreads/CMakeLists.txt @@ -16,9 +16,9 @@ message(STATUS "Added PThreads NVECTOR benchmark") -sundials_add_nvector_benchmark(nvector_pthreads_benchmark +sundials_add_nvector_benchmark( + nvector_pthreads_benchmark SOURCES test_nvector_performance_pthreads.c SUNDIALS_TARGETS sundials_nvecpthreads LINK_LIBRARIES Threads::Threads - INSTALL_SUBDIR nvector/pthreads - ) + INSTALL_SUBDIR nvector/pthreads) diff --git a/benchmarks/nvector/raja/CMakeLists.txt b/benchmarks/nvector/raja/CMakeLists.txt index 4fe8575dbe..9c37eb6d99 100644 --- a/benchmarks/nvector/raja/CMakeLists.txt +++ b/benchmarks/nvector/raja/CMakeLists.txt @@ -37,13 +37,12 @@ foreach(backend ${SUNDIALS_RAJA_BACKENDS}) continue() endif() - sundials_add_nvector_benchmark(${example_target} + sundials_add_nvector_benchmark( + ${example_target} SOURCES test_nvector_performance_raja.cpp - SUNDIALS_TARGETS sundials_nvecraja - ) + SUNDIALS_TARGETS sundials_nvecraja) - target_compile_definitions(${example_target} - PRIVATE ${_defines}) + target_compile_definitions(${example_target} PRIVATE ${_defines}) if(backend MATCHES "CUDA") set_target_properties(${example_target} PROPERTIES LINKER_LANGUAGE CXX) @@ -54,6 +53,6 @@ foreach(backend ${SUNDIALS_RAJA_BACKENDS}) endif() install(TARGETS ${example_target} - DESTINATION "${BENCHMARKS_INSTALL_PATH}/benchmarks/nvector/raja") + DESTINATION "${BENCHMARKS_INSTALL_PATH}/benchmarks/nvector/raja") endforeach() diff --git a/benchmarks/nvector/serial/CMakeLists.txt b/benchmarks/nvector/serial/CMakeLists.txt index 01bb5c1c54..edac263500 100644 --- a/benchmarks/nvector/serial/CMakeLists.txt +++ b/benchmarks/nvector/serial/CMakeLists.txt @@ -16,8 +16,8 @@ message(STATUS "Added Serial NVECTOR benchmark") -sundials_add_nvector_benchmark(nvector_serial_benchmark +sundials_add_nvector_benchmark( + nvector_serial_benchmark SOURCES test_nvector_performance_serial.c SUNDIALS_TARGETS sundials_nvecserial - INSTALL_SUBDIR nvector/serial - ) + INSTALL_SUBDIR nvector/serial) diff --git a/benchmarks/nvector/sycl/CMakeLists.txt b/benchmarks/nvector/sycl/CMakeLists.txt index 6ff0ffbb9f..b92c202b53 100644 --- a/benchmarks/nvector/sycl/CMakeLists.txt +++ b/benchmarks/nvector/sycl/CMakeLists.txt @@ -16,7 +16,8 @@ message(STATUS "Added SYCL NVECTOR benchmark") set(BENCHMARKS_DIR ${PROJECT_SOURCE_DIR}/benchmarks) -sundials_add_nvector_benchmark(test_nvector_performance_sycl +sundials_add_nvector_benchmark( + test_nvector_performance_sycl SOURCES test_nvector_performance_sycl.cpp SUNDIALS_TARGETS sundials_nvecsycl INSTALL_SUBDIR nvector/sycl) diff --git a/benchmarks/nvector/sycl/test_nvector_performance_sycl.cpp b/benchmarks/nvector/sycl/test_nvector_performance_sycl.cpp index 0691d036e4..52b12e966f 100644 --- a/benchmarks/nvector/sycl/test_nvector_performance_sycl.cpp +++ b/benchmarks/nvector/sycl/test_nvector_performance_sycl.cpp @@ -118,7 +118,7 @@ int main(int argc, char* argv[]) ? "Yes" : "No") << std::endl; - std::cout << " suports usm shared allocations? " + std::cout << " supports usm shared allocations? " << (dev.get_info<::sycl::info::device::usm_shared_allocations>() ? "Yes" : "No") diff --git a/benchmarks/nvector/test_nvector_performance.c b/benchmarks/nvector/test_nvector_performance.c index 36224a9c38..c0f0870da2 100644 --- a/benchmarks/nvector/test_nvector_performance.c +++ b/benchmarks/nvector/test_nvector_performance.c @@ -43,7 +43,7 @@ static void time_stats(N_Vector X, double* times, int start, int ntimes, int print_time = 0; /* flag for printing timing data */ int nwarmups = 1; /* number of extra tests to perform and ignore in average */ -#if defined(SUNDIALS_HAVE_POSIX_TIMERS) && defined(_POSIX_TIMERS) +#if defined(SUNDIALS_HAVE_POSIX_TIMERS) time_t base_time_tv_sec = 0; /* Base time; makes time values returned by get_time easier to read when printed since they will be zero @@ -2723,7 +2723,7 @@ void SetNumWarmups(int num_warmups) void SetTiming(int onoff, int myid) { -#if defined(SUNDIALS_HAVE_POSIX_TIMERS) && defined(_POSIX_TIMERS) +#if defined(SUNDIALS_HAVE_POSIX_TIMERS) struct timespec spec; clock_gettime(CLOCK_MONOTONIC, &spec); base_time_tv_sec = spec.tv_sec; @@ -2812,7 +2812,7 @@ void rand_realtype_constraints(sunrealtype* data, sunindextype len) static double get_time(void) { double time; -#if defined(SUNDIALS_HAVE_POSIX_TIMERS) && defined(_POSIX_TIMERS) +#if defined(SUNDIALS_HAVE_POSIX_TIMERS) struct timespec spec; clock_gettime(CLOCK_MONOTONIC, &spec); time = (double)(spec.tv_sec - base_time_tv_sec) + diff --git a/benchmarks/nvector/test_nvector_performance.h b/benchmarks/nvector/test_nvector_performance.h index 23af102a8e..74528b749d 100644 --- a/benchmarks/nvector/test_nvector_performance.h +++ b/benchmarks/nvector/test_nvector_performance.h @@ -17,7 +17,7 @@ #include -/* define constatnts */ +/* define constants */ #define NEG_ONE SUN_RCONST(-1.0) #define ZERO SUN_RCONST(0.0) #define ONE SUN_RCONST(1.0) diff --git a/cmake/SUNDIALSConfig.cmake.in b/cmake/SUNDIALSConfig.cmake.in index 9cbe9e6c23..a5f255f035 100644 --- a/cmake/SUNDIALSConfig.cmake.in +++ b/cmake/SUNDIALSConfig.cmake.in @@ -108,6 +108,16 @@ if("@ENABLE_KOKKOS_KERNELS@" AND NOT TARGET Kokkos::kokkoskernels) find_dependency(KokkosKernels PATHS "@KokkosKernels_DIR@") endif() +if("@ENABLE_LAPACK@" AND NOT TARGET LAPACK::LAPACK) + # For some reason find_dependency does not find the libraries if the variables + # below are internal rather than CACHE variables + set(BLAS_LIBRARIES "@BLAS_LIBRARIES@" CACHE "FILEPATH" "BLAS libraries") + set(BLAS_LINKER_FLAGS "@BLAS_LINKER_FLAGS@" CACHE "STRING" "BLAS linker flags") + set(LAPACK_LIBRARIES "@LAPACK_LIBRARIES@" CACHE "FILEPATH" "LAPACK libraries") + set(LAPACK_LINKER_FLAGS "@LAPACK_LINKER_FLAGS@" CACHE "STRING" "LAPACK linker flags") + find_dependency(LAPACK) +endif() + if("@ENABLE_PETSC@" AND NOT TARGET SUNDIALS::PETSC) add_library(SUNDIALS::PETSC INTERFACE IMPORTED) target_link_libraries(SUNDIALS::PETSC INTERFACE "@PETSC_LIBRARIES@") @@ -161,10 +171,8 @@ if("@ENABLE_RAJA@" AND NOT TARGET RAJA) find_dependency(RAJA PATHS "@RAJA_DIR@") endif() -if("@ENABLE_TRILINOS@" AND NOT TARGET SUNDIALS::TRILINOS) - add_library(SUNDIALS::TRILINOS INTERFACE IMPORTED) - target_link_libraries(SUNDIALS::TRILINOS INTERFACE "@Trilinos_LIBRARIES@") - set_target_properties(SUNDIALS::TRILINOS PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "@Trilinos_INCLUDE_DIRS@") +if("@ENABLE_TRILINOS@" AND NOT TARGET Tpetra::all_libs) + find_dependency(Trilinos COMPONENTS Tpetra PATHS "@Trilinos_DIR@") endif() if("@ENABLE_XBRAID@" AND NOT TARGET SUNDIALS::XBRAID) diff --git a/cmake/SundialsBuildOptionsPost.cmake b/cmake/SundialsBuildOptionsPost.cmake index 01d6969bc6..6d05b9846e 100644 --- a/cmake/SundialsBuildOptionsPost.cmake +++ b/cmake/SundialsBuildOptionsPost.cmake @@ -11,13 +11,13 @@ # SPDX-License-Identifier: BSD-3-Clause # SUNDIALS Copyright End # --------------------------------------------------------------- -# SUNDIALS build options that are interepreted after all other +# SUNDIALS build options that are interpreted after all other # CMake configuration. # --------------------------------------------------------------- # --------------------------------------------------------------- -# Option to use specialized fused kernels in the packages. -# Currently only available in CVODE. +# Option to use specialized fused kernels in the packages. Currently only +# available in CVODE. # --------------------------------------------------------------- if(ENABLE_CUDA OR ENABLE_HIP) @@ -26,9 +26,11 @@ else() set(CUDA_OR_HIP FALSE) endif() -sundials_option(SUNDIALS_BUILD_PACKAGE_FUSED_KERNELS BOOL "Build specialized fused GPU kernels" OFF - DEPENDS_ON BUILD_CVODE CUDA_OR_HIP - DEPENDS_ON_THROW_ERROR) +sundials_option( + SUNDIALS_BUILD_PACKAGE_FUSED_KERNELS BOOL + "Build specialized fused GPU kernels" OFF + DEPENDS_ON BUILD_CVODE CUDA_OR_HIP + DEPENDS_ON_THROW_ERROR) # --------------------------------------------------------------- # Options to enable/disable build for NVECTOR modules. @@ -38,81 +40,100 @@ sundials_option(SUNDIALS_BUILD_PACKAGE_FUSED_KERNELS BOOL "Build specialized fus set(BUILD_NVECTOR_SERIAL TRUE) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_SERIAL") -sundials_option(BUILD_NVECTOR_CUDA BOOL "Build the NVECTOR_CUDA module (requires CUDA)" ON - DEPENDS_ON ENABLE_CUDA CMAKE_CUDA_COMPILER - ADVANCED) +sundials_option( + BUILD_NVECTOR_CUDA BOOL "Build the NVECTOR_CUDA module (requires CUDA)" ON + DEPENDS_ON ENABLE_CUDA CMAKE_CUDA_COMPILER + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_CUDA") -sundials_option(BUILD_NVECTOR_HIP BOOL "Build the NVECTOR_HIP module (requires HIP)" ON - DEPENDS_ON ENABLE_HIP - ADVANCED) +sundials_option( + BUILD_NVECTOR_HIP BOOL "Build the NVECTOR_HIP module (requires HIP)" ON + DEPENDS_ON ENABLE_HIP + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_HIP") -sundials_option(BUILD_NVECTOR_SYCL BOOL "Build the NVECTOR_SYCL module (requires SYCL)" ON - DEPENDS_ON ENABLE_SYCL - ADVANCED) +sundials_option( + BUILD_NVECTOR_SYCL BOOL "Build the NVECTOR_SYCL module (requires SYCL)" ON + DEPENDS_ON ENABLE_SYCL + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_SYCL") -sundials_option(BUILD_NVECTOR_MANYVECTOR BOOL "Build the NVECTOR_MANYVECTOR module" ON - ADVANCED) +sundials_option(BUILD_NVECTOR_MANYVECTOR BOOL + "Build the NVECTOR_MANYVECTOR module" ON ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_MANYVECTOR") -sundials_option(BUILD_NVECTOR_MPIMANYVECTOR BOOL "Build the NVECTOR_MPIMANYVECTOR module (requires MPI)" ON - DEPENDS_ON ENABLE_MPI MPI_C_FOUND - ADVANCED) +sundials_option( + BUILD_NVECTOR_MPIMANYVECTOR BOOL + "Build the NVECTOR_MPIMANYVECTOR module (requires MPI)" ON + DEPENDS_ON ENABLE_MPI MPI_C_FOUND + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_MPIMANYVECTOR") -sundials_option(BUILD_NVECTOR_MPIPLUSX BOOL "Build the NVECTOR_MPIPLUSX module (requires MPI)" ON - DEPENDS_ON ENABLE_MPI MPI_C_FOUND BUILD_NVECTOR_MPIMANYVECTOR - ADVANCED) +sundials_option( + BUILD_NVECTOR_MPIPLUSX BOOL "Build the NVECTOR_MPIPLUSX module (requires MPI)" + ON + DEPENDS_ON ENABLE_MPI MPI_C_FOUND BUILD_NVECTOR_MPIMANYVECTOR + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_MPIPLUSX") -sundials_option(BUILD_NVECTOR_PARALLEL BOOL "Build the NVECTOR_PARALLEL module (requires MPI)" ON - DEPENDS_ON ENABLE_MPI MPI_C_FOUND - ADVANCED) +sundials_option( + BUILD_NVECTOR_PARALLEL BOOL "Build the NVECTOR_PARALLEL module (requires MPI)" + ON + DEPENDS_ON ENABLE_MPI MPI_C_FOUND + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_PARALLEL") -sundials_option(BUILD_NVECTOR_OPENMP BOOL "Build the NVECTOR_OPENMP module" ON - DEPENDS_ON ENABLE_OPENMP - ADVANCED) +sundials_option( + BUILD_NVECTOR_OPENMP BOOL "Build the NVECTOR_OPENMP module" ON + DEPENDS_ON ENABLE_OPENMP + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_OPENMP") -sundials_option(BUILD_NVECTOR_OPENMPDEV BOOL "Build the NVECTOR_OPENMPDEV module" ON - DEPENDS_ON ENABLE_OPENMP_DEVICE OPENMP_SUPPORTS_DEVICE_OFFLOADING - ADVANCED) +sundials_option( + BUILD_NVECTOR_OPENMPDEV BOOL "Build the NVECTOR_OPENMPDEV module" ON + DEPENDS_ON ENABLE_OPENMP_DEVICE OPENMP_SUPPORTS_DEVICE_OFFLOADING + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_OPENMPDEV") -sundials_option(BUILD_NVECTOR_PARHYP BOOL "Build the NVECTOR_PARHYP module (requires hypre)" ON - DEPENDS_ON ENABLE_HYPRE HYPRE_WORKS - ADVANCED) +sundials_option( + BUILD_NVECTOR_PARHYP BOOL "Build the NVECTOR_PARHYP module (requires hypre)" + ON + DEPENDS_ON ENABLE_HYPRE HYPRE_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_PARHYP") -sundials_option(BUILD_NVECTOR_PETSC BOOL "Build the NVECTOR_PETSC module (requires PETSc)" ON - DEPENDS_ON ENABLE_PETSC PETSC_WORKS - ADVANCED) +sundials_option( + BUILD_NVECTOR_PETSC BOOL "Build the NVECTOR_PETSC module (requires PETSc)" ON + DEPENDS_ON ENABLE_PETSC PETSC_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_PETSC") -sundials_option(BUILD_NVECTOR_PTHREADS BOOL "Build the NVECTOR_PTHREADS module" ON - DEPENDS_ON ENABLE_PTHREAD - ADVANCED) +sundials_option( + BUILD_NVECTOR_PTHREADS BOOL "Build the NVECTOR_PTHREADS module" ON + DEPENDS_ON ENABLE_PTHREAD + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_PTHREADS") -sundials_option(BUILD_NVECTOR_RAJA BOOL "Build the NVECTOR_RAJA module (requires RAJA)" ON - DEPENDS_ON ENABLE_RAJA - ADVANCED) +sundials_option( + BUILD_NVECTOR_RAJA BOOL "Build the NVECTOR_RAJA module (requires RAJA)" ON + DEPENDS_ON ENABLE_RAJA + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_RAJA") -sundials_option(BUILD_NVECTOR_TRILINOS BOOL "Build the NVECTOR_TRILINOS module (requires Trilinos)" ON - DEPENDS_ON ENABLE_TRILINOS Trilinos_WORKS - ADVANCED) +sundials_option( + BUILD_NVECTOR_TRILINOS BOOL + "Build the NVECTOR_TRILINOS module (requires Trilinos)" ON + DEPENDS_ON ENABLE_TRILINOS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_TRILINOS") -sundials_option(BUILD_NVECTOR_KOKKOS BOOL "Build the NVECTOR_KOKKOS module (requires Kokkos)" ON - DEPENDS_ON ENABLE_KOKKOS KOKKOS_WORKS - ADVANCED) +sundials_option( + BUILD_NVECTOR_KOKKOS BOOL "Build the NVECTOR_KOKKOS module (requires Kokkos)" + ON + DEPENDS_ON ENABLE_KOKKOS KOKKOS_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_NVECTOR_KOKKOS") - # --------------------------------------------------------------- # Options to enable/disable build for SUNMATRIX modules. # --------------------------------------------------------------- @@ -129,34 +150,47 @@ set(_COMPATIBLE_INDEX_SIZE FALSE) if(SUNDIALS_INDEX_SIZE MATCHES "32") set(_COMPATIBLE_INDEX_SIZE TRUE) endif() -sundials_option(BUILD_SUNMATRIX_CUSPARSE BOOL "Build the SUNMATRIX_CUSPARSE module (requires CUDA and 32-bit indexing)" ON - DEPENDS_ON ENABLE_CUDA CMAKE_CUDA_COMPILER _COMPATIBLE_INDEX_SIZE BUILD_NVECTOR_CUDA - ADVANCED) +sundials_option( + BUILD_SUNMATRIX_CUSPARSE BOOL + "Build the SUNMATRIX_CUSPARSE module (requires CUDA and 32-bit indexing)" ON + DEPENDS_ON ENABLE_CUDA CMAKE_CUDA_COMPILER _COMPATIBLE_INDEX_SIZE + BUILD_NVECTOR_CUDA + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNMATRIX_CUSPARSE") -sundials_option(BUILD_SUNMATRIX_GINKGO BOOL "Build the SUNMATRIX_GINKGO module (requires Ginkgo)" ON - DEPENDS_ON ENABLE_GINKGO GINKGO_WORKS - ADVANCED) +sundials_option( + BUILD_SUNMATRIX_GINKGO BOOL + "Build the SUNMATRIX_GINKGO module (requires Ginkgo)" ON + DEPENDS_ON ENABLE_GINKGO GINKGO_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNMATRIX_GINKGO") -sundials_option(BUILD_SUNMATRIX_KOKKOSDENSE BOOL "Build the SUNMATRIX_KOKKOSDENSE module" ON - DEPENDS_ON ENABLE_KOKKOS KOKKOS_WORKS ENABLE_KOKKOS_KERNELS KOKKOS_KERNELS_WORKS - ADVANCED) +sundials_option( + BUILD_SUNMATRIX_KOKKOSDENSE BOOL "Build the SUNMATRIX_KOKKOSDENSE module" ON + DEPENDS_ON ENABLE_KOKKOS KOKKOS_WORKS ENABLE_KOKKOS_KERNELS + KOKKOS_KERNELS_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNMATRIX_KOKKOSDENSE") -sundials_option(BUILD_SUNMATRIX_MAGMADENSE BOOL "Build the SUNMATRIX_MAGMADENSE module (requires MAGMA)" ON - DEPENDS_ON ENABLE_MAGMA MAGMA_WORKS - ADVANCED) +sundials_option( + BUILD_SUNMATRIX_MAGMADENSE BOOL + "Build the SUNMATRIX_MAGMADENSE module (requires MAGMA)" ON + DEPENDS_ON ENABLE_MAGMA MAGMA_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNMATRIX_MAGMADENSE") -sundials_option(BUILD_SUNMATRIX_ONEMKLDENSE BOOL "Build the SUNMATRIX_ONEMKLDENSE module (requires oneMKL)" ON - DEPENDS_ON ENABLE_ONEMKL ONEMKL_WORKS - ADVANCED) +sundials_option( + BUILD_SUNMATRIX_ONEMKLDENSE BOOL + "Build the SUNMATRIX_ONEMKLDENSE module (requires oneMKL)" ON + DEPENDS_ON ENABLE_ONEMKL ONEMKL_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNMATRIX_ONEMKLDENSE") -sundials_option(BUILD_SUNMATRIX_SLUNRLOC BOOL "Build the SUNMATRIX_SLUNRLOC module (requires SuperLU_DIST)" ON - DEPENDS_ON ENABLE_SUPERLUDIST SUPERLUDIST_WORKS - ADVANCED) +sundials_option( + BUILD_SUNMATRIX_SLUNRLOC BOOL + "Build the SUNMATRIX_SLUNRLOC module (requires SuperLU_DIST)" ON + DEPENDS_ON ENABLE_SUPERLUDIST SUPERLUDIST_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNMATRIX_SLUNRLOC") # --------------------------------------------------------------- @@ -179,57 +213,76 @@ list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_SPGMR") set(BUILD_SUNLINSOL_SPTFQMR TRUE) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_SPTFQMR") -sundials_option(BUILD_SUNLINSOL_CUSOLVERSP BOOL "Build the SUNLINSOL_CUSOLVERSP module (requires CUDA and 32-bit indexing)" ON - DEPENDS_ON ENABLE_CUDA CMAKE_CUDA_COMPILER BUILD_NVECTOR_CUDA BUILD_SUNMATRIX_CUSPARSE - ADVANCED) +sundials_option( + BUILD_SUNLINSOL_CUSOLVERSP BOOL + "Build the SUNLINSOL_CUSOLVERSP module (requires CUDA and 32-bit indexing)" ON + DEPENDS_ON ENABLE_CUDA CMAKE_CUDA_COMPILER BUILD_NVECTOR_CUDA + BUILD_SUNMATRIX_CUSPARSE + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_CUSOLVERSP") -sundials_option(BUILD_SUNLINSOL_GINKGO BOOL "Build the SUNLINSOL_GINKGO module (requires Ginkgo)" ON - DEPENDS_ON ENABLE_GINKGO GINKGO_WORKS - ADVANCED) +sundials_option( + BUILD_SUNLINSOL_GINKGO BOOL + "Build the SUNLINSOL_GINKGO module (requires Ginkgo)" ON + DEPENDS_ON ENABLE_GINKGO GINKGO_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_GINKGO") -sundials_option(BUILD_SUNLINSOL_KLU BOOL "Build the SUNLINSOL_KLU module (requires KLU)" ON - DEPENDS_ON ENABLE_KLU KLU_WORKS - ADVANCED) +sundials_option( + BUILD_SUNLINSOL_KLU BOOL "Build the SUNLINSOL_KLU module (requires KLU)" ON + DEPENDS_ON ENABLE_KLU KLU_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_KLU") -sundials_option(BUILD_SUNLINSOL_KOKKOSDENSE BOOL "Build the SUNLINSOL_KOKKOSDENSE module" ON - DEPENDS_ON ENABLE_KOKKOS KOKKOS_WORKS ENABLE_KOKKOS_KERNELS KOKKOS_KERNELS_WORKS - ADVANCED) +sundials_option( + BUILD_SUNLINSOL_KOKKOSDENSE BOOL "Build the SUNLINSOL_KOKKOSDENSE module" ON + DEPENDS_ON ENABLE_KOKKOS KOKKOS_WORKS ENABLE_KOKKOS_KERNELS + KOKKOS_KERNELS_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_KOKKOSDENSE") -sundials_option(BUILD_SUNLINSOL_LAPACKBAND BOOL "Build the SUNLINSOL_LAPACKBAND module (requires LAPACK)" ON - DEPENDS_ON ENABLE_LAPACK LAPACK_WORKS - ADVANCED) +sundials_option( + BUILD_SUNLINSOL_LAPACKBAND BOOL + "Build the SUNLINSOL_LAPACKBAND module (requires LAPACK)" ON + DEPENDS_ON ENABLE_LAPACK + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_LAPACKBAND") -sundials_option(BUILD_SUNLINSOL_LAPACKDENSE BOOL "Build the SUNLINSOL_LAPACKDENSE module (requires LAPACK)" ON - DEPENDS_ON ENABLE_LAPACK LAPACK_WORKS - ADVANCED) +sundials_option( + BUILD_SUNLINSOL_LAPACKDENSE BOOL + "Build the SUNLINSOL_LAPACKDENSE module (requires LAPACK)" ON + DEPENDS_ON ENABLE_LAPACK + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_LAPACKDENSE") -sundials_option(BUILD_SUNLINSOL_MAGMADENSE BOOL "Build the SUNLINSOL_MAGMADENSE module (requires MAGMA)" ON - DEPENDS_ON ENABLE_MAGMA MAGMA_WORKS - ADVANCED) +sundials_option( + BUILD_SUNLINSOL_MAGMADENSE BOOL + "Build the SUNLINSOL_MAGMADENSE module (requires MAGMA)" ON + DEPENDS_ON ENABLE_MAGMA MAGMA_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_MAGMADENSE") -sundials_option(BUILD_SUNLINSOL_ONEMKLDENSE BOOL "Build the SUNLINSOL_ONEMKLDENSE module (requires oneMKL)" ON - DEPENDS_ON ENABLE_ONEMKL ONEMKL_WORKS - ADVANCED) +sundials_option( + BUILD_SUNLINSOL_ONEMKLDENSE BOOL + "Build the SUNLINSOL_ONEMKLDENSE module (requires oneMKL)" ON + DEPENDS_ON ENABLE_ONEMKL ONEMKL_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_ONEMKLDENSE") -sundials_option(BUILD_SUNLINSOL_SUPERLUDIST BOOL "Build the SUNLINSOL_SUPERLUDIST module (requires SUPERLUDIST)" ON - DEPENDS_ON ENABLE_SUPERLUDIST SUPERLUDIST_WORKS BUILD_SUNMATRIX_SLUNRLOC - ADVANCED) +sundials_option( + BUILD_SUNLINSOL_SUPERLUDIST BOOL + "Build the SUNLINSOL_SUPERLUDIST module (requires SUPERLUDIST)" ON + DEPENDS_ON ENABLE_SUPERLUDIST SUPERLUDIST_WORKS BUILD_SUNMATRIX_SLUNRLOC + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_SUPERLUDIST") -sundials_option(BUILD_SUNLINSOL_SUPERLUMT BOOL "Build the SUNLINSOL_SUPERLUMT module (requires SUPERLUMT)" ON - DEPENDS_ON ENABLE_SUPERLUMT SUPERLUMT_WORKS - ADVANCED) +sundials_option( + BUILD_SUNLINSOL_SUPERLUMT BOOL + "Build the SUNLINSOL_SUPERLUMT module (requires SUPERLUMT)" ON + DEPENDS_ON ENABLE_SUPERLUMT SUPERLUMT_WORKS + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNLINSOL_SUPERLUMT") - # --------------------------------------------------------------- # Options to enable/disable build for SUNNONLINSOL modules. # --------------------------------------------------------------- @@ -240,7 +293,9 @@ list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNNONLINSOL_NEWTON") set(BUILD_SUNNONLINSOL_FIXEDPOINT TRUE) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNNONLINSOL_FIXEDPOINT") -sundials_option(BUILD_SUNNONLINSOL_PETSCSNES BOOL "Build the SUNNONLINSOL_PETSCSNES module (requires PETSc)" ON - DEPENDS_ON ENABLE_PETSC PETSC_FOUND - ADVANCED) +sundials_option( + BUILD_SUNNONLINSOL_PETSCSNES BOOL + "Build the SUNNONLINSOL_PETSCSNES module (requires PETSc)" ON + DEPENDS_ON ENABLE_PETSC PETSC_FOUND + ADVANCED) list(APPEND SUNDIALS_BUILD_LIST "BUILD_SUNNONLINSOL_PETSCSNES") diff --git a/cmake/SundialsBuildOptionsPre.cmake b/cmake/SundialsBuildOptionsPre.cmake index 3c3f6b16cb..96e17ea087 100644 --- a/cmake/SundialsBuildOptionsPre.cmake +++ b/cmake/SundialsBuildOptionsPre.cmake @@ -11,7 +11,7 @@ # SPDX-License-Identifier: BSD-3-Clause # SUNDIALS Copyright End # --------------------------------------------------------------- -# SUNDIALS build options that are interepreted prior to any +# SUNDIALS build options that are interpreted prior to any # other CMake configuration. # --------------------------------------------------------------- @@ -33,7 +33,8 @@ endif() set(DOCSTR "single, double, or extended") sundials_option(SUNDIALS_PRECISION STRING "${DOCSTR}" "DOUBLE") string(TOUPPER ${SUNDIALS_PRECISION} _upper_SUNDIALS_PRECISION) -force_variable(SUNDIALS_PRECISION STRING "${DOCSTR}" ${_upper_SUNDIALS_PRECISION}) +force_variable(SUNDIALS_PRECISION STRING "${DOCSTR}" + ${_upper_SUNDIALS_PRECISION}) # --------------------------------------------------------------- # Option to specify index type @@ -65,7 +66,9 @@ set(DOCSTR "Build with simulation profiling capabilities enabled") sundials_option(SUNDIALS_BUILD_WITH_PROFILING BOOL "${DOCSTR}" OFF) if(SUNDIALS_BUILD_WITH_PROFILING) - message(WARNING "SUNDIALS built with profiling turned on, performance may be affected.") + message( + WARNING + "SUNDIALS built with profiling turned on, performance may be affected.") endif() # --------------------------------------------------------------- @@ -78,24 +81,35 @@ else() set(_default_err_checks ON) endif() -set(DOCSTR "Build with error checking enabled/disabled. Enabling error checks may affect performance.") -sundials_option(SUNDIALS_ENABLE_ERROR_CHECKS BOOL "${DOCSTR}" ${_default_err_checks}) +set(DOCSTR + "Build with error checking enabled/disabled. Enabling error checks may affect performance." +) +sundials_option(SUNDIALS_ENABLE_ERROR_CHECKS BOOL "${DOCSTR}" + ${_default_err_checks}) if(SUNDIALS_ENABLE_ERROR_CHECKS) message(STATUS "SUNDIALS error checking enabled") - message(WARNING "SUNDIALS is being built with extensive error checks, performance may be affected.") + message( + WARNING + "SUNDIALS is being built with extensive error checks, performance may be affected." + ) endif() # --------------------------------------------------------------- # Option to enable logging # --------------------------------------------------------------- -set(DOCSTR "Build with logging capabilities enabled (0 = no logging, 1 = errors, 2 = +warnings, 3 = +info, 4 = +debug, 5 = +extras") +set(DOCSTR + "Build with logging capabilities enabled (0 = no logging, 1 = errors, 2 = +warnings, 3 = +info, 4 = +debug, 5 = +extras" +) sundials_option(SUNDIALS_LOGGING_LEVEL STRING "${DOCSTR}" 2 OPTIONS "0;1;2;3;4;5") if(SUNDIALS_LOGGING_LEVEL GREATER_EQUAL 3) message(STATUS "SUNDIALS logging level set to ${SUNDIALS_LOGGING_LEVEL}") - message(WARNING "SUNDIALS built with additional logging turned on, performance may be affected.") + message( + WARNING + "SUNDIALS built with additional logging turned on, performance may be affected." + ) endif() # --------------------------------------------------------------- @@ -103,9 +117,11 @@ endif() # --------------------------------------------------------------- if(UNIX) - sundials_option(SUNDIALS_MATH_LIBRARY PATH "Which math library (e.g., libm) to link to" "-lm" ADVANCED) + sundials_option(SUNDIALS_MATH_LIBRARY PATH + "Which math library (e.g., libm) to link to" "-lm" ADVANCED) else() - sundials_option(SUNDIALS_MATH_LIBRARY PATH "Which math library (e.g., libm) to link to" "" ADVANCED) + sundials_option(SUNDIALS_MATH_LIBRARY PATH + "Which math library (e.g., libm) to link to" "" ADVANCED) endif() # all executables will be linked against the math library set(EXE_EXTRA_LINK_LIBS "${SUNDIALS_MATH_LIBRARY}") @@ -119,15 +135,16 @@ sundials_option(BUILD_SHARED_LIBS BOOL "Build shared libraries" ON) # Make sure we build at least one type of libraries if(NOT BUILD_STATIC_LIBS AND NOT BUILD_SHARED_LIBS) - message(FATAL_ERROR "Both static and shared library generation were disabled.") + message( + FATAL_ERROR "Both static and shared library generation were disabled.") endif() # --------------------------------------------------------------- # Options to enable SUNDIALS packages and modules # --------------------------------------------------------------- -# For each SUNDIALS package available (i.e. for which we have the -# sources), give the user the option of enabling/disabling it. +# For each SUNDIALS package available (i.e. for which we have the sources), give +# the user the option of enabling/disabling it. if(IS_DIRECTORY "${SUNDIALS_SOURCE_DIR}/src/arkode") sundials_option(BUILD_ARKODE BOOL "Build the ARKODE library" ON) @@ -182,7 +199,10 @@ sundials_option(BUILD_FORTRAN_MODULE_INTERFACE BOOL "${DOCSTR}" OFF) if(BUILD_FORTRAN_MODULE_INTERFACE) # F2003 interface only supports double precision if(NOT (SUNDIALS_PRECISION MATCHES "DOUBLE")) - message(FATAL_ERROR "F2003 interface is not compatible with ${SUNDIALS_PRECISION} precision") + message( + FATAL_ERROR + "F2003 interface is not compatible with ${SUNDIALS_PRECISION} precision" + ) endif() # Allow a user to set where the Fortran modules will be installed @@ -196,7 +216,10 @@ endif() sundials_option(BUILD_BENCHMARKS BOOL "Build the SUNDIALS benchmark suite" OFF) -sundials_option(BENCHMARKS_INSTALL_PATH PATH "Output directory for installing benchmark executables" "${CMAKE_INSTALL_PREFIX}/benchmarks") +sundials_option( + BENCHMARKS_INSTALL_PATH PATH + "Output directory for installing benchmark executables" + "${CMAKE_INSTALL_PREFIX}/benchmarks") # --------------------------------------------------------------- # Options for CMake config installation @@ -210,14 +233,14 @@ sundials_option(SUNDIALS_INSTALL_CMAKEDIR STRING "${DOCSTR}" # Options to enable compiler warnings, address sanitizer # --------------------------------------------------------------- -sundials_option(ENABLE_ALL_WARNINGS BOOL - "Enable all compiler warnings" OFF ADVANCED) +sundials_option(ENABLE_ALL_WARNINGS BOOL "Enable all compiler warnings" OFF + ADVANCED) sundials_option(ENABLE_WARNINGS_AS_ERRORS BOOL - "Enable compiler warnings as errors" OFF ADVANCED) + "Enable compiler warnings as errors" OFF ADVANCED) -sundials_option(ENABLE_ADDRESS_SANITIZER BOOL - "Enable address sanitizer" OFF ADVANCED) +sundials_option(ENABLE_ADDRESS_SANITIZER BOOL "Enable address sanitizer" OFF + ADVANCED) # --------------------------------------------------------------- # Options to enable SUNDIALS debugging @@ -225,111 +248,127 @@ sundials_option(ENABLE_ADDRESS_SANITIZER BOOL # List of debugging options (used to add preprocessor directives) set(_SUNDIALS_DEBUG_OPTIONS - SUNDIALS_DEBUG - SUNDIALS_DEBUG_ASSERT - SUNDIALS_DEBUG_CUDA_LASTERROR - SUNDIALS_DEBUG_HIP_LASTERROR - SUNDIALS_DEBUG_PRINTVEC) + SUNDIALS_DEBUG SUNDIALS_DEBUG_ASSERT SUNDIALS_DEBUG_CUDA_LASTERROR + SUNDIALS_DEBUG_HIP_LASTERROR SUNDIALS_DEBUG_PRINTVEC) sundials_option(SUNDIALS_DEBUG BOOL - "Enable additional debugging output and options" OFF - ADVANCED) + "Enable additional debugging output and options" OFF ADVANCED) if(SUNDIALS_DEBUG AND SUNDIALS_LOGGING_LEVEL LESS 4) set(DOCSTR "SUNDIALS_DEBUG=ON forced the logging level to 4") message(STATUS "${DOCSTR}") - set(SUNDIALS_LOGGING_LEVEL "4" CACHE STRING "${DOCSTR}" FORCE) + set(SUNDIALS_LOGGING_LEVEL + "4" + CACHE STRING "${DOCSTR}" FORCE) endif() -sundials_option(SUNDIALS_DEBUG_ASSERT BOOL - "Enable assert when debugging" OFF +sundials_option( + SUNDIALS_DEBUG_ASSERT BOOL "Enable assert when debugging" OFF DEPENDS_ON SUNDIALS_DEBUG ADVANCED) -sundials_option(SUNDIALS_DEBUG_CUDA_LASTERROR BOOL +sundials_option( + SUNDIALS_DEBUG_CUDA_LASTERROR BOOL "Enable CUDA last error checks when debugging" OFF DEPENDS_ON SUNDIALS_DEBUG ENABLE_CUDA ADVANCED) -sundials_option(SUNDIALS_DEBUG_HIP_LASTERROR BOOL +sundials_option( + SUNDIALS_DEBUG_HIP_LASTERROR BOOL "Enable HIP last error checks when debugging" OFF DEPENDS_ON SUNDIALS_DEBUG ENABLE_HIP ADVANCED) -sundials_option(SUNDIALS_DEBUG_PRINTVEC BOOL - "Enable vector printing when debugging" OFF +sundials_option( + SUNDIALS_DEBUG_PRINTVEC BOOL "Enable vector printing when debugging" OFF DEPENDS_ON SUNDIALS_DEBUG ADVANCED) if(SUNDIALS_DEBUG_PRINTVEC AND SUNDIALS_LOGGING_LEVEL LESS 5) set(DOCSTR "SUNDIALS_DEBUG_PRINTVEC=ON forced the logging level to 5") message(STATUS "${DOCSTR}") - set(SUNDIALS_LOGGING_LEVEL "5" CACHE STRING "${DOCSTR}" FORCE) + set(SUNDIALS_LOGGING_LEVEL + "5" + CACHE STRING "${DOCSTR}" FORCE) endif() # --------------------------------------------------------------- # Options for SUNDIALS external # --------------------------------------------------------------- -sundials_option(SUNDIALS_ENABLE_EXTERNAL_ADDONS BOOL +sundials_option( + SUNDIALS_ENABLE_EXTERNAL_ADDONS BOOL "Enables including EXTERNALLY MAINTAINED addons in the SUNDIALS build." OFF) if(SUNDIALS_ENABLE_EXTERNAL_ADDONS) - message(WARNING "SUNDIALS_ENABLE_EXTERNAL_ADDONS=TRUE. External addons are not maintained by the SUNDIALS team. Use at your own risk.") + message( + WARNING + "SUNDIALS_ENABLE_EXTERNAL_ADDONS=TRUE. External addons are not maintained by the SUNDIALS team. Use at your own risk." + ) endif() # --------------------------------------------------------------- # Options for SUNDIALS testing # --------------------------------------------------------------- -sundials_option(SUNDIALS_TEST_FLOAT_PRECISION STRING +sundials_option( + SUNDIALS_TEST_FLOAT_PRECISION STRING "Precision for floating point comparisons (number of digits)" "-1" ADVANCED) -sundials_option(SUNDIALS_TEST_INTEGER_PRECISION STRING +sundials_option( + SUNDIALS_TEST_INTEGER_PRECISION STRING "Precision for integer comparisons (percent difference)" "-1" ADVANCED) sundials_option(SUNDIALS_TEST_OUTPUT_DIR PATH - "Location to write testing output files" "" ADVANCED) + "Location to write testing output files" "" ADVANCED) sundials_option(SUNDIALS_TEST_ANSWER_DIR PATH - "Location of testing answer files" "" ADVANCED) + "Location of testing answer files" "" ADVANCED) sundials_option(SUNDIALS_TEST_PROFILE BOOL - "Use Caliper to profile SUNDIALS tests" OFF ADVANCED) + "Use Caliper to profile SUNDIALS tests" OFF ADVANCED) -sundials_option(SUNDIALS_TEST_NODIFF BOOL +sundials_option( + SUNDIALS_TEST_NODIFF BOOL "Disable output comparison in the regression test suite" OFF ADVANCED) -sundials_option(SUNDIALS_TEST_CONTAINER_EXE PATH - "Path to docker or podman" "" ADVANCED) +sundials_option(SUNDIALS_TEST_CONTAINER_EXE PATH "Path to docker or podman" "" + ADVANCED) -sundials_option(SUNDIALS_TEST_CONTAINER_RUN_EXTRA_ARGS STRING - "Extra arguments to pass to docker/podman run command" "--tls-verify=false" ADVANCED) +sundials_option( + SUNDIALS_TEST_CONTAINER_RUN_EXTRA_ARGS STRING + "Extra arguments to pass to docker/podman run command" "--tls-verify=false" + ADVANCED) -sundials_option(SUNDIALS_TEST_CONTAINER_MNT STRING +sundials_option( + SUNDIALS_TEST_CONTAINER_MNT STRING "Path to project root inside the container" "/sundials" ADVANCED) # Include development examples in regression tests sundials_option(SUNDIALS_TEST_DEVTESTS BOOL - "Include development tests in make test" OFF ADVANCED) + "Include development tests in make test" OFF ADVANCED) # Include unit tests in regression tests -sundials_option(SUNDIALS_TEST_UNITTESTS BOOL - "Include unit tests in make test" OFF ADVANCED) +sundials_option(SUNDIALS_TEST_UNITTESTS BOOL "Include unit tests in make test" + OFF ADVANCED) # Include googletest unit tests in regression tests -sundials_option(SUNDIALS_TEST_ENABLE_GTEST BOOL - "Disable GTest unit tests" ON ADVANCED) +sundials_option(SUNDIALS_TEST_ENABLE_GTEST BOOL "Disable GTest unit tests" ON + ADVANCED) -sundials_option(SUNDIALS_DEV_IWYU BOOL - "Enable include-what-you-use" OFF ADVANCED) +sundials_option(SUNDIALS_DEV_IWYU BOOL "Enable include-what-you-use" OFF + ADVANCED) -sundials_option(SUNDIALS_DEV_CLANG_TIDY BOOL - "Enable clang-tidy" OFF ADVANCED) +sundials_option(SUNDIALS_DEV_CLANG_TIDY BOOL "Enable clang-tidy" OFF ADVANCED) -sundials_option(SUNDIALS_SCHEDULER_COMMAND STRING "Job scheduler command to use to launch SUNDIALS MPI tests" "" ADVANCED) +sundials_option( + SUNDIALS_SCHEDULER_COMMAND STRING + "Job scheduler command to use to launch SUNDIALS MPI tests" "" ADVANCED) -sundials_option(SUNDIALS_CALIPER_OUTPUT_DIR PATH "Location to write caliper output files" "" ADVANCED) +sundials_option(SUNDIALS_CALIPER_OUTPUT_DIR PATH + "Location to write caliper output files" "" ADVANCED) -sundials_option(SUNDIALS_BENCHMARK_NUM_CPUS STRING "Number of CPU cores to run benchmarks with" "40" ADVANCED) +sundials_option(SUNDIALS_BENCHMARK_NUM_CPUS STRING + "Number of CPU cores to run benchmarks with" "40" ADVANCED) -sundials_option(SUNDIALS_BENCHMARK_NUM_GPUS STRING "Number of GPUs to run benchmarks with" "4" ADVANCED) +sundials_option(SUNDIALS_BENCHMARK_NUM_GPUS STRING + "Number of GPUs to run benchmarks with" "4" ADVANCED) diff --git a/cmake/SundialsDeprecated.cmake b/cmake/SundialsDeprecated.cmake index 385a11361b..017e867c73 100644 --- a/cmake/SundialsDeprecated.cmake +++ b/cmake/SundialsDeprecated.cmake @@ -18,8 +18,10 @@ if(DEFINED F2003_INTERFACE_ENABLE) message(DEPRECATION "The CMake option F2003_INTERFACE_ENABLE is deprecated. " - "Use BUILD_FORTRAN_MODULE_INTERFACE instead.") - set(BUILD_FORTRAN_MODULE_INTERFACE ${F2003_INTERFACE_ENABLE} CACHE BOOL "Enable Fortran 2003 module interfaces") + "Use BUILD_FORTRAN_MODULE_INTERFACE instead.") + set(BUILD_FORTRAN_MODULE_INTERFACE + ${F2003_INTERFACE_ENABLE} + CACHE BOOL "Enable Fortran 2003 module interfaces") endif() unset(F2003_INTERFACE_ENABLE CACHE) @@ -30,115 +32,146 @@ unset(F2003_INTERFACE_ENABLE CACHE) if(DEFINED MPI_ENABLE) message(DEPRECATION "The CMake option MPI_ENABLE is deprecated. " - "Use ENABLE_MPI instead.") - set(ENABLE_MPI ${MPI_ENABLE} CACHE BOOL "Enable MPI support" FORCE) + "Use ENABLE_MPI instead.") + set(ENABLE_MPI + ${MPI_ENABLE} + CACHE BOOL "Enable MPI support" FORCE) unset(MPI_ENABLE CACHE) endif() if(DEFINED OPENMP_ENABLE) message(DEPRECATION "The CMake option OPENMP_ENABLE is deprecated. " - "Use ENABLE_OPENMP instead.") - set(ENABLE_OPENMP ${OPENMP_ENABLE} CACHE BOOL "Enable OpenMP support" FORCE) + "Use ENABLE_OPENMP instead.") + set(ENABLE_OPENMP + ${OPENMP_ENABLE} + CACHE BOOL "Enable OpenMP support" FORCE) unset(OPENMP_ENABLE CACHE) endif() if(DEFINED OPENMP_DEVICE_ENABLE) message(DEPRECATION "The CMake option OPENMP_DEVICE_ENABLE is deprecated. " - "Use ENABLE_OPENMP_DEVICE instead.") - set(ENABLE_OPENMP_DEVICE ${OPENMP_DEVICE_ENABLE} CACHE BOOL - "Enable OpenMP device offloading support" FORCE) + "Use ENABLE_OPENMP_DEVICE instead.") + set(ENABLE_OPENMP_DEVICE + ${OPENMP_DEVICE_ENABLE} + CACHE BOOL "Enable OpenMP device offloading support" FORCE) unset(OPENMP_DEVICE_ENABLE CACHE) endif() if(DEFINED SKIP_OPENMP_DEVICE_CHECK) - message(DEPRECATION "The CMake option SKIP_OPENMP_DEVICE_CHECK is deprecated. " - "Use OPENMP_DEVICE_WORKS instead.") - set(OPENMP_DEVICE_WORKS ${SKIP_OPENMP_DEVICE_CHECK} CACHE BOOL - "Skip the compiler check for OpenMP device offloading" FORCE) + message( + DEPRECATION "The CMake option SKIP_OPENMP_DEVICE_CHECK is deprecated. " + "Use OPENMP_DEVICE_WORKS instead.") + set(OPENMP_DEVICE_WORKS + ${SKIP_OPENMP_DEVICE_CHECK} + CACHE BOOL "Skip the compiler check for OpenMP device offloading" FORCE) unset(SKIP_OPENMP_DEVICE_CHECK CACHE) endif() if(DEFINED PTHREAD_ENABLE) message(DEPRECATION "The CMake option PTHREAD_ENABLE is deprecated. " - "Use ENABLE_PTHREAD instead") - set(ENABLE_PTHREAD ${PTHREAD_ENABLE} CACHE BOOL "Enable Pthreads support" FORCE) + "Use ENABLE_PTHREAD instead") + set(ENABLE_PTHREAD + ${PTHREAD_ENABLE} + CACHE BOOL "Enable Pthreads support" FORCE) unset(PTHREAD_ENABLE CACHE) endif() if(DEFINED CUDA_ENABLE) message(DEPRECATION "The CMake option CUDA_ENABLE is deprecated. " - "Use ENABLE_CUDA instead.") - set(ENABLE_CUDA ${CUDA_ENABLE} CACHE BOOL "Enable CUDA support" FORCE) + "Use ENABLE_CUDA instead.") + set(ENABLE_CUDA + ${CUDA_ENABLE} + CACHE BOOL "Enable CUDA support" FORCE) unset(CUDA_ENABLE CACHE) endif() if(DEFINED LAPACK_ENABLE) message(DEPRECATION "The CMake option LAPACK_ENABLE is deprecated. " - "Use ENABLE_LAPACK instead.") - set(ENABLE_LAPACK ${LAPACK_ENABLE} CACHE BOOL "Enable LAPACK support" FORCE) + "Use ENABLE_LAPACK instead.") + set(ENABLE_LAPACK + ${LAPACK_ENABLE} + CACHE BOOL "Enable LAPACK support" FORCE) unset(LAPACK_ENABLE CACHE) endif() if(DEFINED SUPERLUDIST_ENABLE) message(DEPRECATION "The CMake option SUPERLUDIST_ENABLE is deprecated. " - "Use ENABLE_SUPERLUDIST instead.") - set(ENABLE_SUPERLUDIST ${SUPERLUDIST_ENABLE} CACHE BOOL "Enable SuperLU_DIST support" FORCE) + "Use ENABLE_SUPERLUDIST instead.") + set(ENABLE_SUPERLUDIST + ${SUPERLUDIST_ENABLE} + CACHE BOOL "Enable SuperLU_DIST support" FORCE) unset(SUPERLUDIST_ENABLE CACHE) endif() # Deprecated with SUNDIALS 6.4.0 if(DEFINED SUPERLUDIST_LIBRARY_DIR) message(DEPRECATION "The CMake option SUPERLUDIST_LIBRARY_DIR is deprecated. " - "Use SUPERLUDIST_DIR instead.") - set(SUPERLUDIST_DIR "${SUPERLUDIST_LIBRARY_DIR}/../" CACHE BOOL "SuperLU_DIST root directory" FORCE) + "Use SUPERLUDIST_DIR instead.") + set(SUPERLUDIST_DIR + "${SUPERLUDIST_LIBRARY_DIR}/../" + CACHE BOOL "SuperLU_DIST root directory" FORCE) unset(SUPERLUDIST_LIBRARY_DIR CACHE) endif() if(DEFINED SUPERLUDIST_INCLUDE_DIR) message(DEPRECATION "The CMake option SUPERLUDIST_INCLUDE_DIR is deprecated. " - "Use SUPERLUDIST_INCLUDE_DIRS instead.") - set(SUPERLUDIST_INCLUDE_DIRS "${SUPERLUDIST_INCLUDE_DIR}" CACHE BOOL "SuperLU_DIST include directoroes" FORCE) + "Use SUPERLUDIST_INCLUDE_DIRS instead.") + set(SUPERLUDIST_INCLUDE_DIRS + "${SUPERLUDIST_INCLUDE_DIR}" + CACHE BOOL "SuperLU_DIST include directoroes" FORCE) unset(SUPERLUDIST_INCLUDE_DIR CACHE) endif() if(DEFINED SUPERLUMT_ENABLE) message(DEPRECATION "The CMake option SUPERLUMT_ENABLE is deprecated. " - "Use ENABLE_SUPERLUMT instead.") - set(ENABLE_SUPERLUMT ${SUPERLUMT_ENABLE} CACHE BOOL "Enable SuperLU_MT support" FORCE) + "Use ENABLE_SUPERLUMT instead.") + set(ENABLE_SUPERLUMT + ${SUPERLUMT_ENABLE} + CACHE BOOL "Enable SuperLU_MT support" FORCE) unset(SUPERLUMT_ENABLE CACHE) endif() if(DEFINED KLU_ENABLE) message(DEPRECATION "The CMake option KLU_ENABLE is deprecated. " - "Use ENABLE_KLU instead.") - set(ENABLE_KLU ${KLU_ENABLE} CACHE BOOL "Enable KLU support" FORCE) + "Use ENABLE_KLU instead.") + set(ENABLE_KLU + ${KLU_ENABLE} + CACHE BOOL "Enable KLU support" FORCE) unset(KLU_ENABLE CACHE) endif() if(DEFINED HYPRE_ENABLE) message(DEPRECATION "The CMake option HYPRE_ENABLE is deprecated. " - "Use ENABLE_HYPRE instead.") - set(ENABLE_HYPRE ${HYPRE_ENABLE} CACHE BOOL "Enable HYPRE support" FORCE) + "Use ENABLE_HYPRE instead.") + set(ENABLE_HYPRE + ${HYPRE_ENABLE} + CACHE BOOL "Enable HYPRE support" FORCE) unset(HYPRE_ENABLE CACHE) endif() if(DEFINED PETSC_ENABLE) message(DEPRECATION "The CMake option PETSC_ENABLE is deprecated. " - "Use ENABLE_PETSC instead.") - set(ENABLE_PETSC ${PETSC_ENABLE} CACHE BOOL "Enable PETSC support" FORCE) + "Use ENABLE_PETSC instead.") + set(ENABLE_PETSC + ${PETSC_ENABLE} + CACHE BOOL "Enable PETSC support" FORCE) unset(PETSC_ENABLE CACHE) endif() if(DEFINED Trilinos_ENABLE) message(DEPRECATION "The CMake option Trilinos_ENABLE is deprecated. " - "Use ENABLE_TRILINOS instead.") - set(ENABLE_TRILINOS ${Trilinos_ENABLE} CACHE BOOL "Enable Trilinos support" FORCE) + "Use ENABLE_TRILINOS instead.") + set(ENABLE_TRILINOS + ${Trilinos_ENABLE} + CACHE BOOL "Enable Trilinos support" FORCE) unset(Trilinos_ENABLE CACHE) endif() if(DEFINED RAJA_ENABLE) message(DEPRECATION "The CMake option RAJA_ENABLE is deprecated. " - "Use ENABLE_RAJA instead.") - set(ENABLE_RAJA ${RAJA_ENABLE} CACHE BOOL "Enable RAJA support" FORCE) + "Use ENABLE_RAJA instead.") + set(ENABLE_RAJA + ${RAJA_ENABLE} + CACHE BOOL "Enable RAJA support" FORCE) unset(RAJA_ENABLE CACHE) endif() @@ -147,10 +180,12 @@ endif() # if(DEFINED CUDA_ARCH) - print_warning("The CMake option CUDA_ARCH is deprecated. " - "Use CMAKE_CUDA_ARCHITECTURES instead.") + message(DEPRECATION "The CMake option CUDA_ARCH is deprecated. " + "Use CMAKE_CUDA_ARCHITECTURES instead.") # convert sm_** to just ** string(REGEX MATCH "[0-9]+" arch_name "${CUDA_ARCH}") - set(CMAKE_CUDA_ARCHITECTURES ${arch_name} CACHE STRING "CUDA Architectures" FORCE) + set(CMAKE_CUDA_ARCHITECTURES + ${arch_name} + CACHE STRING "CUDA Architectures" FORCE) unset(CUDA_ARCH) endif() diff --git a/cmake/SundialsExampleOptions.cmake b/cmake/SundialsExampleOptions.cmake index 40d692771f..8d54377bd6 100644 --- a/cmake/SundialsExampleOptions.cmake +++ b/cmake/SundialsExampleOptions.cmake @@ -22,9 +22,16 @@ sundials_option(EXAMPLES_ENABLE_C BOOL "Build SUNDIALS C examples" ON) # Some TPLs only have C++ examples. Default the C++ examples to ON if any of # these are enabled on the initial configuration pass. -if (ENABLE_TRILINOS OR ENABLE_SUPERLUDIST OR ENABLE_XBRAID OR ENABLE_HIP OR - ENABLE_MAGMA OR ENABLE_SYCL OR ENABLE_ONEMKL OR ENABLE_RAJA OR ENABLE_GINKGO OR - ENABLE_KOKKOS) +if(ENABLE_TRILINOS + OR ENABLE_SUPERLUDIST + OR ENABLE_XBRAID + OR ENABLE_HIP + OR ENABLE_MAGMA + OR ENABLE_SYCL + OR ENABLE_ONEMKL + OR ENABLE_RAJA + OR ENABLE_GINKGO + OR ENABLE_KOKKOS) sundials_option(EXAMPLES_ENABLE_CXX BOOL "Build SUNDIALS C++ examples" ON) else() sundials_option(EXAMPLES_ENABLE_CXX BOOL "Build SUNDIALS C++ examples" OFF) @@ -34,24 +41,30 @@ endif() # Options for Fortran Examples # ----------------------------------------------------------------------------- -# F2003 examples (on by default) are an option only if the -# Fortran 2003 interface is enabled. +# F2003 examples (on by default) are an option only if the Fortran 2003 +# interface is enabled. set(DOCSTR "Build SUNDIALS Fortran 2003 examples") if(BUILD_FORTRAN_MODULE_INTERFACE) - set(EXAMPLES_ENABLE_F2003 ON CACHE BOOL "${DOCSTR}") + set(EXAMPLES_ENABLE_F2003 + ON + CACHE BOOL "${DOCSTR}") # Fortran 2003 examples only support double precision if(EXAMPLES_ENABLE_F2003 AND (NOT (SUNDIALS_PRECISION MATCHES "DOUBLE"))) - message(WARNING "F2003 examples are not compatible with ${SUNDIALS_PRECISION} precision. " - "Setting EXAMPLES_ENABLE_F2003 to OFF.") + message( + WARNING + "F2003 examples are not compatible with ${SUNDIALS_PRECISION} precision. " + "Setting EXAMPLES_ENABLE_F2003 to OFF.") force_variable(EXAMPLES_ENABLE_F2003 BOOL "${DOCSTR}" OFF) endif() else() # set back to OFF (in case it was ON) if(EXAMPLES_ENABLE_F2003) - message(WARNING "EXAMPLES_ENABLE_F2003 is ON but BUILD_FORTRAN_MODULE_INTERFACE is OFF. " - "Setting EXAMPLES_ENABLE_F2003 to OFF.") + message( + WARNING + "EXAMPLES_ENABLE_F2003 is ON but BUILD_FORTRAN_MODULE_INTERFACE is OFF. " + "Setting EXAMPLES_ENABLE_F2003 to OFF.") force_variable(EXAMPLES_ENABLE_F2003 BOOL "${DOCSTR}" OFF) endif() @@ -71,25 +84,33 @@ sundials_option(EXAMPLES_ENABLE_CUDA BOOL "Build SUNDIALS CUDA examples" ON # Enable installing examples by default sundials_option(EXAMPLES_INSTALL BOOL "Install SUNDIALS examples" ON) -sundials_option(EXAMPLES_INSTALL_PATH PATH "Output directory for installing example files" "${CMAKE_INSTALL_PREFIX}/examples") +sundials_option( + EXAMPLES_INSTALL_PATH PATH "Output directory for installing example files" + "${CMAKE_INSTALL_PREFIX}/examples") # If examples are to be exported, check where we should install them. if(EXAMPLES_INSTALL AND NOT EXAMPLES_INSTALL_PATH) - message(WARNING "The example installation path is empty. Example installation " - "path was reset to its default value") - set(EXAMPLES_INSTALL_PATH "${CMAKE_INSTALL_PREFIX}/examples" CACHE STRING - "Output directory for installing example files" FORCE) + message( + WARNING "The example installation path is empty. Example installation " + "path was reset to its default value") + set(EXAMPLES_INSTALL_PATH + "${CMAKE_INSTALL_PREFIX}/examples" + CACHE STRING "Output directory for installing example files" FORCE) endif() # ----------------------------------------------------------------------------- # Internal variables. # ----------------------------------------------------------------------------- -if(EXAMPLES_ENABLE_C OR - EXAMPLES_ENABLE_CXX OR - EXAMPLES_ENABLE_CUDA OR - EXAMPLES_ENABLE_F2003) - set(_BUILD_EXAMPLES TRUE CACHE INTERNAL "") +if(EXAMPLES_ENABLE_C + OR EXAMPLES_ENABLE_CXX + OR EXAMPLES_ENABLE_CUDA + OR EXAMPLES_ENABLE_F2003) + set(_BUILD_EXAMPLES + TRUE + CACHE INTERNAL "") else() - set(_BUILD_EXAMPLES FALSE CACHE INTERNAL "") + set(_BUILD_EXAMPLES + FALSE + CACHE INTERNAL "") endif() diff --git a/cmake/SundialsIndexSize.cmake b/cmake/SundialsIndexSize.cmake index 6498637bd3..f8f4836144 100644 --- a/cmake/SundialsIndexSize.cmake +++ b/cmake/SundialsIndexSize.cmake @@ -24,8 +24,9 @@ include(CheckTypeSize) if(SUNDIALS_INDEX_SIZE MATCHES "64") set(SUNDIALS_CINDEX_TYPE "") - # if the user specified an index type use it, otherwise try the standard options - if (SUNDIALS_INDEX_TYPE) + # if the user specified an index type use it, otherwise try the standard + # options + if(SUNDIALS_INDEX_TYPE) set(POSSIBLE_INT64 ${SUNDIALS_INDEX_TYPE}) else() set(POSSIBLE_INT64 int64_t;__int64;long long;long) @@ -42,8 +43,11 @@ if(SUNDIALS_INDEX_SIZE MATCHES "64") endforeach() if(NOT SUNDIALS_CINDEX_TYPE) - message(FATAL_ERROR "No integer type of size 8 was found. Tried " - "${POSSIBLE_INT64}. Try setting the advanced option SUNDIALS_INDEX_TYPE.") + message( + FATAL_ERROR + "No integer type of size 8 was found. Tried " + "${POSSIBLE_INT64}. Try setting the advanced option SUNDIALS_INDEX_TYPE." + ) endif() # set Fortran integer size too @@ -51,8 +55,9 @@ if(SUNDIALS_INDEX_SIZE MATCHES "64") elseif(SUNDIALS_INDEX_SIZE MATCHES "32") set(SUNDIALS_CINDEX_TYPE "") - # if the user specified an index type use it, otherwise try the standard options - if (SUNDIALS_INDEX_TYPE) + # if the user specified an index type use it, otherwise try the standard + # options + if(SUNDIALS_INDEX_TYPE) set(POSSIBLE_INT32 ${SUNDIALS_INDEX_TYPE}) else() set(POSSIBLE_INT32 int32_t;int;long) @@ -69,8 +74,11 @@ elseif(SUNDIALS_INDEX_SIZE MATCHES "32") endforeach() if(NOT SUNDIALS_CINDEX_TYPE) - message(FATAL_ERROR "No integer type of size 4 was found. Tried " - "${POSSIBLE_INT32}. Try setting the advanced option SUNDIALS_INDEX_TYPE.") + message( + FATAL_ERROR + "No integer type of size 4 was found. Tried " + "${POSSIBLE_INT32}. Try setting the advanced option SUNDIALS_INDEX_TYPE." + ) endif() # set Fortran integer size too diff --git a/cmake/SundialsSetupCXX.cmake b/cmake/SundialsSetupCXX.cmake index 32a6a9b003..187e597d40 100644 --- a/cmake/SundialsSetupCXX.cmake +++ b/cmake/SundialsSetupCXX.cmake @@ -19,24 +19,21 @@ enable_language(CXX) set(CXX_FOUND TRUE) # --------------------------------------------------------------- -# Option to specify the C++ standard SUNDIALS will use. Defined -# here so it is set in the same configuration pass as the C++ -# compiler and related options. +# Option to specify the C++ standard SUNDIALS will use. Defined here so it is +# set in the same configuration pass as the C++ compiler and related options. # --------------------------------------------------------------- # Do not allow decaying to previous standards -- generates error if the standard # is not supported -sundials_option(CMAKE_CXX_STANDARD_REQUIRED BOOL - "Require C++ standard version" ON) +sundials_option(CMAKE_CXX_STANDARD_REQUIRED BOOL "Require C++ standard version" + ON) if(ENABLE_SYCL) set(DOCSTR "The C++ standard to use if C++ is enabled (17, 20)") - sundials_option(CMAKE_CXX_STANDARD STRING "${DOCSTR}" "17" - OPTIONS "17;20") + sundials_option(CMAKE_CXX_STANDARD STRING "${DOCSTR}" "17" OPTIONS "17;20") else() set(DOCSTR "The C++ standard to use if C++ is enabled (14, 17, 20)") - sundials_option(CMAKE_CXX_STANDARD STRING "${DOCSTR}" "14" - OPTIONS "14;17;20") + sundials_option(CMAKE_CXX_STANDARD STRING "${DOCSTR}" "14" OPTIONS "14;17;20") endif() message(STATUS "CXX standard set to ${CMAKE_CXX_STANDARD}") @@ -44,7 +41,7 @@ set(DOCSTR "Enable C++ compiler specific extensions") sundials_option(CMAKE_CXX_EXTENSIONS BOOL "${DOCSTR}" ON) message(STATUS "C++ extensions set to ${CMAKE_CXX_EXTENSIONS}") -# SYCL requries C++17 +# SYCL requires C++17 if(ENABLE_SYCL AND (CMAKE_CXX_STANDARD LESS "17")) message(SEND_ERROR "CMAKE_CXX_STANDARD must be >= 17 because ENABLE_SYCL=ON") endif() diff --git a/cmake/SundialsSetupCompilers.cmake b/cmake/SundialsSetupCompilers.cmake index 7790f61609..ab008f1ad0 100644 --- a/cmake/SundialsSetupCompilers.cmake +++ b/cmake/SundialsSetupCompilers.cmake @@ -21,16 +21,16 @@ include(SundialsIndexSize) # =============================================================== -# Platform specifc settings +# Platform specific settings # =============================================================== if(WIN32) - # Under Windows, add compiler directive to inhibit warnings - # about use of unsecure functions. + # Under Windows, add compiler directive to inhibit warnings about use of + # insecure functions. add_compile_definitions(_CRT_SECURE_NO_WARNINGS) - # Under Windows, we need to have dll and exe files in the - # same directory to run the test suite properly. + # Under Windows, we need to have dll and exe files in the same directory to + # run the test suite properly. set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") @@ -38,7 +38,8 @@ endif() if(APPLE) # Allow undefined symbols that will be resolved by a user program. - set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS} -undefined dynamic_lookup") + set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS + "${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS} -undefined dynamic_lookup") endif() # =============================================================== @@ -50,18 +51,20 @@ if(BUILD_SHARED_LIBS) # use, i.e. don't skip the full RPATH for the build tree set(CMAKE_SKIP_BUILD_RPATH FALSE) - # when building, don't use the install RPATH already - # (but later on when installing) + # when building, don't use the install RPATH already (but later on when + # installing) set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_FULL_LIBDIR}") set(CMAKE_INSTALL_NAME_DIR "${CMAKE_INSTALL_FULL_LIBDIR}") - # add the automatically determined parts of the RPATH - # which point to directories outside the build tree to the install RPATH + # add the automatically determined parts of the RPATH which point to + # directories outside the build tree to the install RPATH set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - # the RPATH to be used when installing, but only if it's not a system directory - list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_FULL_LIBDIR}" isSystemDir) + # the RPATH to be used when installing, but only if it's not a system + # directory + list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES + "${CMAKE_INSTALL_FULL_LIBDIR}" isSystemDir) if("${isSystemDir}" STREQUAL "-1") set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_FULL_LIBDIR}") endif() @@ -76,39 +79,49 @@ endif() if(ENABLE_ALL_WARNINGS) message(STATUS "Enabling all compiler warnings") + # Some warning flags are not supported by all compilers so ignore unknown + # flags with -Wno-unknown-warning-option. Ironically, this is not supported by + # some compilers. + set(WARNING_FLAGS + "-Wno-unknown-warning-option -Wall -Wpedantic -Wextra -Wshadow \ +-Wwrite-strings -Wcast-align -Wdisabled-optimization -Wvla -Walloca \ +-Wduplicated-cond -Wduplicated-branches") + # TODO(SBR): Try to add -Wredundant-decls once SuperLU version is updated in + # CI tests + # Avoid numerous warnings from printf if(SUNDIALS_PRECISION MATCHES "EXTENDED") - set(CMAKE_C_FLAGS "-Wdouble-promotion ${CMAKE_C_FLAGS}") - set(CMAKE_CXX_FLAGS "-Wdouble-promotion ${CMAKE_CXX_FLAGS}") + set(WARNING_FLAGS "-Wdouble-promotion ${WARNING_FLAGS}") endif() - if((SUNDIALS_PRECISION MATCHES "DOUBLE") AND (SUNDIALS_INDEX_SIZE MATCHES "32")) - set(CMAKE_C_FLAGS "-Wconversion -Wno-sign-conversion ${CMAKE_C_FLAGS}") - set(CMAKE_CXX_FLAGS "-Wconversion -Wno-sign-conversion ${CMAKE_CXX_FLAGS}") + if((SUNDIALS_PRECISION MATCHES "DOUBLE") AND (SUNDIALS_INDEX_SIZE MATCHES "32" + )) + set(WARNING_FLAGS "-Wconversion -Wno-sign-conversion ${WARNING_FLAGS}") endif() # Avoid numerous warnings from SWIG generated functions if(NOT BUILD_FORTRAN_MODULE_INTERFACE) - set(CMAKE_C_FLAGS "-Wmissing-declarations -Wcast-qual ${CMAKE_C_FLAGS}") - set(CMAKE_CXX_FLAGS "-Wmissing-declarations -Wcast-qual ${CMAKE_CXX_FLAGS}") + set(WARNING_FLAGS "-Wmissing-declarations -Wcast-qual ${WARNING_FLAGS}") endif() - set(CMAKE_C_FLAGS "-Wall -Wpedantic -Wextra -Wshadow ${CMAKE_C_FLAGS}") - set(CMAKE_CXX_FLAGS "-Wall -Wpedantic -Wextra -Wshadow ${CMAKE_CXX_FLAGS}") + set(CMAKE_C_FLAGS "${WARNING_FLAGS} ${CMAKE_C_FLAGS}") + set(CMAKE_CXX_FLAGS "${WARNING_FLAGS} ${CMAKE_CXX_FLAGS}") # TODO(DJG): Add -fcheck=all,no-pointer,no-recursion once Jenkins is updated # to use gfortran > 5.5 which segfaults with -fcheck=array-temps,bounds,do,mem # no- options were added in gfortran 6 # - # Exclude run-time pointer checks (no-pointer) because passing null objects - # to SUNDIALS functions (e.g., sunmat => null() to SetLinearSolver) causes a + # Exclude run-time pointer checks (no-pointer) because passing null objects to + # SUNDIALS functions (e.g., sunmat => null() to SetLinearSolver) causes a # run-time error with this check # # Exclude checks for subroutines and functions not marked as recursive # (no-recursion) e.g., ark_brusselator1D_task_local_nls_f2003 calls # SUNNonlinsolFree from within a custom nonlinear solver implementation of # SUNNonlinsolFree which causes a run-time error with this check - set(CMAKE_Fortran_FLAGS "-Wall -Wpedantic -Wno-unused-dummy-argument -Wno-c-binding-type -ffpe-summary=none ${CMAKE_Fortran_FLAGS}") + set(CMAKE_Fortran_FLAGS + "-Wall -Wpedantic -Wno-unused-dummy-argument -Wno-c-binding-type -ffpe-summary=none ${CMAKE_Fortran_FLAGS}" + ) endif() if(ENABLE_WARNINGS_AS_ERRORS) @@ -119,19 +132,46 @@ if(ENABLE_WARNINGS_AS_ERRORS) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Werror") endif() +# With clang it is not possible to combine the -fsanitize=address and +# -fsanitize=memory checkers. + if(ENABLE_ADDRESS_SANITIZER) message(STATUS "Enabling address sanitizer") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address -fsanitize=leak -fsanitize=undefined") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fsanitize=leak -fsanitize=undefined") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fsanitize=address -fsanitize=leak -fsanitize=undefined") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fsanitize=address") +endif() + +if(ENABLE_MEMORY_SANITIZER) + message(STATUS "Enabling memory sanitizer") + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=memory") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=memory") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fsanitize=memory") +endif() + +if(ENABLE_LEAK_SANITIZER) + message(STATUS "Enabling leak sanitizer") + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=leak") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=leak") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fsanitize=leak") +endif() + +if(ENABLE_UNDEFINED_BEHAVIOR_SANITIZER) + message(STATUS "Enabling undefined behavior sanitizer") + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fsanitize=undefined") endif() if(SUNDIALS_DEBUG) message(STATUS "Adding debugging preprocessor directives") foreach(debug ${_SUNDIALS_DEBUG_OPTIONS}) - if (${${debug}}) + if(${${debug}}) add_compile_definitions(${debug}) endif() endforeach() @@ -142,8 +182,7 @@ endif() # =============================================================== set(DOCSTR "The C standard to use (99, 11, 17)") -sundials_option(CMAKE_C_STANDARD STRING "${DOCSTR}" "99" - OPTIONS "99;11;17") +sundials_option(CMAKE_C_STANDARD STRING "${DOCSTR}" "99" OPTIONS "99;11;17") message(STATUS "C standard set to ${CMAKE_C_STANDARD}") set(DOCSTR "Enable C compiler specific extensions") @@ -154,7 +193,8 @@ message(STATUS "C extensions set to ${CMAKE_C_EXTENSIONS}") # Check for __builtin_expect # --------------------------------------------------------------- -check_c_source_compiles(" +check_c_source_compiles( + " #include int main(void) { double a = 0.0; @@ -165,14 +205,16 @@ check_c_source_compiles(" printf(\"a=%g\", a); return 0; } -" SUNDIALS_C_COMPILER_HAS_BUILTIN_EXPECT) +" + SUNDIALS_C_COMPILER_HAS_BUILTIN_EXPECT) # --------------------------------------------------------------- # Check for assume related extensions # --------------------------------------------------------------- # gcc >= 13 should have __attribute__((assume)) -check_c_source_compiles(" +check_c_source_compiles( + " #include int main(void) { double a = 0.0; @@ -188,11 +230,13 @@ check_c_source_compiles(" printf(\"a=%g\", a); return 0; } -" SUNDIALS_C_COMPILER_HAS_ATTRIBUTE_ASSUME) +" + SUNDIALS_C_COMPILER_HAS_ATTRIBUTE_ASSUME) # LLVM based compilers should have __builtin_assume if(NOT SUNDIALS_C_COMPILER_HAS_ATTRIBUTE_ASSUME) - check_c_source_compiles(" + check_c_source_compiles( + " #include int main(void) { double a = 0.0; @@ -201,12 +245,15 @@ if(NOT SUNDIALS_C_COMPILER_HAS_ATTRIBUTE_ASSUME) printf(\"a=%g\", a); return 0; } - " SUNDIALS_C_COMPILER_HAS_BUILTIN_ASSUME) + " + SUNDIALS_C_COMPILER_HAS_BUILTIN_ASSUME) endif() # MSVC provides __assume -if(NOT (SUNDIALS_C_COMPILER_HAS_ATTRIBUTE_ASSUME OR SUNDIALS_C_COMPILER_HAS_BUILTIN_ASSUME)) - check_c_source_compiles(" +if(NOT (SUNDIALS_C_COMPILER_HAS_ATTRIBUTE_ASSUME + OR SUNDIALS_C_COMPILER_HAS_BUILTIN_ASSUME)) + check_c_source_compiles( + " #include int main(void) { double a = 0.0; @@ -215,19 +262,22 @@ if(NOT (SUNDIALS_C_COMPILER_HAS_ATTRIBUTE_ASSUME OR SUNDIALS_C_COMPILER_HAS_BUIL printf(\"a=%g\", a); return 0; } - " SUNDIALS_C_COMPILER_HAS_ASSUME) + " + SUNDIALS_C_COMPILER_HAS_ASSUME) endif() # --------------------------------------------------------------- # Check for unused extension # --------------------------------------------------------------- -check_c_source_compiles(" +check_c_source_compiles( + " int main(void) { __attribute__((unused)) double a = 0.0; return 0; } -" SUNDIALS_C_COMPILER_HAS_ATTRIBUTE_UNUSED) +" + SUNDIALS_C_COMPILER_HAS_ATTRIBUTE_UNUSED) # --------------------------------------------------------------- # Check for POSIX timers @@ -236,25 +286,29 @@ include(SundialsPOSIXTimers) if(SUNDIALS_POSIX_TIMERS AND POSIX_TIMERS_NEED_POSIX_C_SOURCE) set(DOCSTR "Value of _POSIX_C_SOURCE") - sundials_option(SUNDIALS_POSIX_C_SOURCE STRING "${DOCSTR}" "200112L" - ADVANCED) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_POSIX_C_SOURCE=${SUNDIALS_POSIX_C_SOURCE}") + sundials_option(SUNDIALS_POSIX_C_SOURCE STRING "${DOCSTR}" "200112L" ADVANCED) + set(CMAKE_C_FLAGS + "${CMAKE_C_FLAGS} -D_POSIX_C_SOURCE=${SUNDIALS_POSIX_C_SOURCE}") endif() - # --------------------------------------------------------------- # Check for deprecated attribute with message # --------------------------------------------------------------- if(WIN32) - set(COMPILER_DEPRECATED_MSG_ATTRIBUTE "__declspec(deprecated(msg))" CACHE INTERNAL "") + set(COMPILER_DEPRECATED_MSG_ATTRIBUTE + "__declspec(deprecated(msg))" + CACHE INTERNAL "") else() - set(COMPILER_DEPRECATED_MSG_ATTRIBUTE "__attribute__ ((__deprecated__(msg)))" CACHE INTERNAL "") + set(COMPILER_DEPRECATED_MSG_ATTRIBUTE + "__attribute__ ((__deprecated__(msg)))" + CACHE INTERNAL "") endif() -check_c_source_compiles(" +check_c_source_compiles( + " #define msg \"test\" ${COMPILER_DEPRECATED_MSG_ATTRIBUTE} int somefunc(void) { return 0; } - int main(void) { return somefunc();}" COMPILER_HAS_DEPRECATED_MSG -) + int main(void) { return somefunc();}" + COMPILER_HAS_DEPRECATED_MSG) # =============================================================== # Fortran settings @@ -282,24 +336,22 @@ endif() # The case to use in the name-mangling scheme sundials_option(SUNDIALS_LAPACK_CASE STRING - "case of LAPACK function names (lower/upper)" - "" - ADVANCED) + "case of LAPACK function names (lower/upper)" "" ADVANCED) # The number of underscores of appended in the name-mangling scheme -sundials_option(SUNDIALS_LAPACK_UNDERSCORES STRING - "number of underscores appended to LAPACK function names (none/one/two)" - "" - ADVANCED) +sundials_option( + SUNDIALS_LAPACK_UNDERSCORES STRING + "number of underscores appended to LAPACK function names (none/one/two)" "" + ADVANCED) # If used, both case and underscores must be set if((NOT SUNDIALS_LAPACK_CASE) AND SUNDIALS_LAPACK_UNDERSCORES) message(FATAL_ERROR "If SUNDIALS_LAPACK_UNDERSCORES is set, " - "SUNDIALS_LAPACK_CASE must also be set.") + "SUNDIALS_LAPACK_CASE must also be set.") endif() if(SUNDIALS_LAPACK_CASE AND (NOT SUNDIALS_LAPACK_UNDERSCORES)) message(FATAL_ERROR "If SUNDIALS_LAPACK_CASE is set, " - "SUNDIALS_LAPACK_UNDERSCORES must also be set.") + "SUNDIALS_LAPACK_UNDERSCORES must also be set.") endif() # Did the user provide a name-mangling scheme? @@ -318,11 +370,15 @@ if(SUNDIALS_LAPACK_CASE AND SUNDIALS_LAPACK_UNDERSCORES) set(LAPACK_MANGLE_MACRO1 "#define SUNDIALS_LAPACK_FUNC(name,NAME) name") set(LAPACK_MANGLE_MACRO2 "#define SUNDIALS_LAPACK_FUNC_(name,NAME) name") elseif(SUNDIALS_LAPACK_UNDERSCORES MATCHES "ONE") - set(LAPACK_MANGLE_MACRO1 "#define SUNDIALS_LAPACK_FUNC(name,NAME) name ## _") - set(LAPACK_MANGLE_MACRO2 "#define SUNDIALS_LAPACK_FUNC_(name,NAME) name ## _") + set(LAPACK_MANGLE_MACRO1 + "#define SUNDIALS_LAPACK_FUNC(name,NAME) name ## _") + set(LAPACK_MANGLE_MACRO2 + "#define SUNDIALS_LAPACK_FUNC_(name,NAME) name ## _") elseif(SUNDIALS_LAPACK_UNDERSCORES MATCHES "TWO") - set(LAPACK_MANGLE_MACRO1 "#define SUNDIALS_LAPACK_FUNC(name,NAME) name ## __") - set(LAPACK_MANGLE_MACRO2 "#define SUNDIALS_LAPACK_FUNC_(name,NAME) name ## __") + set(LAPACK_MANGLE_MACRO1 + "#define SUNDIALS_LAPACK_FUNC(name,NAME) name ## __") + set(LAPACK_MANGLE_MACRO2 + "#define SUNDIALS_LAPACK_FUNC_(name,NAME) name ## __") else() message(FATAL_ERROR "Invalid SUNDIALS_LAPACK_UNDERSCORES option.") endif() @@ -331,11 +387,15 @@ if(SUNDIALS_LAPACK_CASE AND SUNDIALS_LAPACK_UNDERSCORES) set(LAPACK_MANGLE_MACRO1 "#define SUNDIALS_LAPACK_FUNC(name,NAME) NAME") set(LAPACK_MANGLE_MACRO2 "#define SUNDIALS_LAPACK_FUNC_(name,NAME) NAME") elseif(SUNDIALS_LAPACK_UNDERSCORES MATCHES "ONE") - set(LAPACK_MANGLE_MACRO1 "#define SUNDIALS_LAPACK_FUNC(name,NAME) NAME ## _") - set(LAPACK_MANGLE_MACRO2 "#define SUNDIALS_LAPACK_FUNC_(name,NAME) NAME ## _") + set(LAPACK_MANGLE_MACRO1 + "#define SUNDIALS_LAPACK_FUNC(name,NAME) NAME ## _") + set(LAPACK_MANGLE_MACRO2 + "#define SUNDIALS_LAPACK_FUNC_(name,NAME) NAME ## _") elseif(SUNDIALS_LAPACK_UNDERSCORES MATCHES "TWO") - set(LAPACK_MANGLE_MACRO1 "#define SUNDIALS_LAPACK_FUNC(name,NAME) NAME ## __") - set(LAPACK_MANGLE_MACRO2 "#define SUNDIALS_LAPACK_FUNC_(name,NAME) NAME ## __") + set(LAPACK_MANGLE_MACRO1 + "#define SUNDIALS_LAPACK_FUNC(name,NAME) NAME ## __") + set(LAPACK_MANGLE_MACRO2 + "#define SUNDIALS_LAPACK_FUNC_(name,NAME) NAME ## __") else() message(FATAL_ERROR "Invalid SUNDIALS_LAPACK_UNDERSCORES option.") endif() @@ -346,16 +406,13 @@ if(SUNDIALS_LAPACK_CASE AND SUNDIALS_LAPACK_UNDERSCORES) # name-mangling scheme has been manually set set(NEED_FORTRAN_NAME_MANGLING FALSE) - configure_file( - ${PROJECT_SOURCE_DIR}/src/sundials/sundials_lapack_defs.h.in - ${PROJECT_BINARY_DIR}/src/sundials/sundials_lapack_defs.h - ) + configure_file(${PROJECT_SOURCE_DIR}/src/sundials/sundials_lapack_defs.h.in + ${PROJECT_BINARY_DIR}/src/sundials/sundials_lapack_defs.h) endif() # Do we need a Fortran compiler? -if(BUILD_FORTRAN_MODULE_INTERFACE OR - NEED_FORTRAN_NAME_MANGLING) +if(BUILD_FORTRAN_MODULE_INTERFACE OR NEED_FORTRAN_NAME_MANGLING) include(SundialsSetupFortran) endif() @@ -363,17 +420,19 @@ endif() # C++ settings # =============================================================== -if(BUILD_BENCHMARKS OR SUNDIALS_TEST_UNITTESTS OR EXAMPLES_ENABLE_CXX OR - ENABLE_CUDA OR - ENABLE_HIP OR - ENABLE_SYCL OR - ENABLE_RAJA OR - ENABLE_TRILINOS OR - ENABLE_SUPERLUDIST OR - ENABLE_MAGMA OR - ENABLE_GINKGO OR - ENABLE_KOKKOS OR - ENABLE_ADIAK) +if(BUILD_BENCHMARKS + OR SUNDIALS_TEST_UNITTESTS + OR EXAMPLES_ENABLE_CXX + OR ENABLE_CUDA + OR ENABLE_HIP + OR ENABLE_SYCL + OR ENABLE_RAJA + OR ENABLE_TRILINOS + OR ENABLE_SUPERLUDIST + OR ENABLE_MAGMA + OR ENABLE_GINKGO + OR ENABLE_KOKKOS + OR ENABLE_ADIAK) include(SundialsSetupCXX) endif() @@ -416,8 +475,8 @@ endif() # Upper case version of build type string(TOUPPER "${CMAKE_BUILD_TYPE}" _cmake_build_type) -# Make build type specific flag options ADVANCED, -# except for the one corresponding to the current build type +# Make build type specific flag options ADVANCED, except for the one +# corresponding to the current build type foreach(lang ${_SUNDIALS_ENABLED_LANGS}) foreach(build_type DEBUG;RELEASE;RELWITHDEBINFO;MINSIZEREL) if("${_cmake_build_type}" STREQUAL "${build_type}") @@ -431,7 +490,6 @@ foreach(lang ${_SUNDIALS_ENABLED_LANGS}) mark_as_advanced(CLEAR CMAKE_${lang}_COMPILER CMAKE_${lang}_FLAGS) endforeach() - # =============================================================== # Configure compilers for installed examples # =============================================================== @@ -439,14 +497,17 @@ endforeach() foreach(lang ${_SUNDIALS_ENABLED_LANGS}) if(ENABLE_MPI) if(DEFINED MPI_${lang}_COMPILER) - set(_EXAMPLES_${lang}_COMPILER "${MPI_${lang}_COMPILER}" CACHE INTERNAL "${lang} compiler for installed examples") + set(_EXAMPLES_${lang}_COMPILER + "${MPI_${lang}_COMPILER}" + CACHE INTERNAL "${lang} compiler for installed examples") endif() else() - set(_EXAMPLES_${lang}_COMPILER "${CMAKE_${lang}_COMPILER}" CACHE INTERNAL "${lang} compiler for installed examples") + set(_EXAMPLES_${lang}_COMPILER + "${CMAKE_${lang}_COMPILER}" + CACHE INTERNAL "${lang} compiler for installed examples") endif() endforeach() - # =============================================================== # Configure clang-tidy for linting # =============================================================== @@ -456,7 +517,7 @@ set(SUNDIALS_DEV_CLANG_TIDY_DIR ${CMAKE_BINARY_DIR}/clang-tidy/) if(SUNDIALS_DEV_CLANG_TIDY) find_program(CLANG_TIDY_PATH NAMES clang-tidy) if(NOT CLANG_TIDY_PATH) - message(FATAL_ERROR "Could not find the program clang-tidy") + message(FATAL_ERROR "Could not find the program clang-tidy") endif() message(STATUS "Found clang-tidy: ${CLANG_TIDY_PATH}") @@ -465,15 +526,12 @@ if(SUNDIALS_DEV_CLANG_TIDY) set(CMAKE_C_CLANG_TIDY ${CLANG_TIDY_PATH} -format-style='file' --fix) set(CMAKE_CXX_CLANG_TIDY ${CLANG_TIDY_PATH} -format-style='file' --fix) else() - set(CMAKE_C_CLANG_TIDY ${CLANG_TIDY_PATH} - -format-style='file' - --export-fixes=${SUNDIALS_DEV_CLANG_TIDY_DIR}/clang-tidy-fixes.yaml - ) + set(CMAKE_C_CLANG_TIDY + ${CLANG_TIDY_PATH} -format-style='file' + --export-fixes=${SUNDIALS_DEV_CLANG_TIDY_DIR}/clang-tidy-fixes.yaml) set(CMAKE_CXX_CLANG_TIDY - ${CLANG_TIDY_PATH} - -format-style='file' - --export-fixes=${SUNDIALS_DEV_CLANG_TIDY_DIR}/clang-tidy-cxx-fixes.yaml - ) + ${CLANG_TIDY_PATH} -format-style='file' + --export-fixes=${SUNDIALS_DEV_CLANG_TIDY_DIR}/clang-tidy-cxx-fixes.yaml) endif() endif() @@ -483,10 +541,10 @@ if(SUNDIALS_DEV_IWYU) message(FATAL_ERROR "Could not find the program include-what-you-use") endif() message(STATUS "Found IWYU: ${IWYU_PATH}") - set(CMAKE_C_INCLUDE_WHAT_YOU_USE ${IWYU_PATH} - -Xiwyu --mapping_file=${CMAKE_SOURCE_DIR}/scripts/iwyu.imp - -Xiwyu --error_always) - set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${IWYU_PATH} - -Xiwyu --mapping_file=${CMAKE_SOURCE_DIR}/scripts/iwyu.imp - -Xiwyu --error_always) + set(CMAKE_C_INCLUDE_WHAT_YOU_USE + ${IWYU_PATH} -Xiwyu --mapping_file=${CMAKE_SOURCE_DIR}/scripts/iwyu.imp + -Xiwyu --error_always) + set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE + ${IWYU_PATH} -Xiwyu --mapping_file=${CMAKE_SOURCE_DIR}/scripts/iwyu.imp + -Xiwyu --error_always) endif() diff --git a/cmake/SundialsSetupConfig.cmake b/cmake/SundialsSetupConfig.cmake index 482a267ac4..7000a90396 100644 --- a/cmake/SundialsSetupConfig.cmake +++ b/cmake/SundialsSetupConfig.cmake @@ -44,10 +44,9 @@ else() string(TIMESTAMP JOB_START_TIME "%Y%m%d%H%M%S") endif() - # ============================================================================ -# Generate macros and substitution variables related to TPLs -# that SUNDIALS is being built with. +# Generate macros and substitution variables related to TPLs that SUNDIALS is +# being built with. # ============================================================================ # prepare substitution variables for modules that have been built @@ -59,13 +58,14 @@ foreach(_item ${SUNDIALS_BUILD_LIST}) endif() endforeach() -# prepare substitution variable SUNDIALS_${TPL NAME}_ENABLED for sundials_config.h +# prepare substitution variable SUNDIALS_${TPL NAME}_ENABLED for +# sundials_config.h foreach(tpl ${SUNDIALS_TPL_LIST}) set(SUNDIALS_${tpl}_ENABLED TRUE) endforeach() # prepare substitution variable SUNDIALS_TRILINOS_HAVE_MPI for sundials_config.h -if(Trilinos_MPI) +if(ENABLE_MPI) set(SUNDIALS_TRILINOS_HAVE_MPI TRUE) endif() @@ -94,7 +94,5 @@ endif() # Generate the header file and place it in the binary dir. # ============================================================================= -configure_file( - ${PROJECT_SOURCE_DIR}/include/sundials/sundials_config.in - ${PROJECT_BINARY_DIR}/include/sundials/sundials_config.h - ) +configure_file(${PROJECT_SOURCE_DIR}/include/sundials/sundials_config.in + ${PROJECT_BINARY_DIR}/include/sundials/sundials_config.h) diff --git a/cmake/SundialsSetupCuda.cmake b/cmake/SundialsSetupCuda.cmake index f971e691a6..09703288bc 100644 --- a/cmake/SundialsSetupCuda.cmake +++ b/cmake/SundialsSetupCuda.cmake @@ -11,7 +11,7 @@ # SPDX-License-Identifier: BSD-3-Clause # SUNDIALS Copyright End # --------------------------------------------------------------- -# Setup the CUDA languge and CUDA libraries. +# Setup the CUDA language and CUDA libraries. # --------------------------------------------------------------- # =============================================================== @@ -19,9 +19,11 @@ # =============================================================== if(NOT CMAKE_CUDA_HOST_COMPILER) - # If a user did not provide the host compiler, then we - # assume that they want to use the CXX compiler that was set. - set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE FILEPATH "NVCC host compiler") + # If a user did not provide the host compiler, then we assume that they want + # to use the CXX compiler that was set. + set(CMAKE_CUDA_HOST_COMPILER + ${CMAKE_CXX_COMPILER} + CACHE FILEPATH "NVCC host compiler") endif() # =============================================================== @@ -31,18 +33,19 @@ endif() # Do not allow decaying to previous standards -- generates error if the standard # is not supported sundials_option(CMAKE_CUDA_STANDARD_REQUIRED BOOL - "Require C++ standard version" ON) + "Require C++ standard version" ON) set(DOCSTR "The CUDA standard to use if CUDA is enabled (14, 17, 20)") sundials_option(CMAKE_CUDA_STANDARD STRING "${DOCSTR}" "${CMAKE_CXX_STANDARD}" OPTIONS "14;17;20") message(STATUS "CUDA standard set to ${CMAKE_CUDA_STANDARD}") -set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr") +set(CMAKE_CUDA_FLAGS + "${CMAKE_CUDA_FLAGS} --expt-extended-lambda --expt-relaxed-constexpr") -if( (CMAKE_CXX_COMPILER_ID MATCHES GNU) - OR (CMAKE_CXX_COMPILER_ID MATCHES Clang) - AND (CMAKE_SYSTEM_PROCESSOR MATCHES ppc64le) ) +if((CMAKE_CXX_COMPILER_ID MATCHES GNU) + OR (CMAKE_CXX_COMPILER_ID MATCHES Clang) + AND (CMAKE_SYSTEM_PROCESSOR MATCHES ppc64le)) include(CheckCXXCompilerFlag) check_cxx_compiler_flag(-mno-float128 _hasflag) if(_hasflag) @@ -50,6 +53,15 @@ if( (CMAKE_CXX_COMPILER_ID MATCHES GNU) endif() endif() +# =============================================================== +# Prohibit CUDA interface when using extended precision. +# =============================================================== + +if(SUNDIALS_PRECISION MATCHES "EXTENDED") + message( + FATAL_ERROR "CUDA interfaces are incompatible with extended precision.") +endif() + # =============================================================== # Enable CUDA lang and find the CUDA libraries. # =============================================================== @@ -75,15 +87,19 @@ message(STATUS "CUDA Library Directory: ${CUDAToolkit_LIBRARY_DIR}") message(STATUS "CUDA Compile Flags: ${CMAKE_CUDA_FLAGS}") message(STATUS "CUDA Link Flags: ${CMAKE_CUDA_LINK_FLAGS}") message(STATUS "CUDA Link Executable: ${CMAKE_CUDA_LINK_EXECUTABLE}") -message(STATUS "CUDA Separable Compilation: ${CMAKE_CUDA_SEPARABLE_COMPILATION}") - +message( + STATUS "CUDA Separable Compilation: ${CMAKE_CUDA_SEPARABLE_COMPILATION}") # =============================================================== # Configure compiler for installed examples # =============================================================== if(ENABLE_MPI) - set(_EXAMPLES_CUDA_HOST_COMPILER "${MPI_CXX_COMPILER}" CACHE INTERNAL "${lang} compiler for installed examples") + set(_EXAMPLES_CUDA_HOST_COMPILER + "${MPI_CXX_COMPILER}" + CACHE INTERNAL "${lang} compiler for installed examples") else() - set(_EXAMPLES_CUDA_HOST_COMPILER "${CMAKE_CUDA_HOST_COMPILER}" CACHE INTERNAL "${lang} compiler for installed examples") + set(_EXAMPLES_CUDA_HOST_COMPILER + "${CMAKE_CUDA_HOST_COMPILER}" + CACHE INTERNAL "${lang} compiler for installed examples") endif() diff --git a/cmake/SundialsSetupFortran.cmake b/cmake/SundialsSetupFortran.cmake index de9beea905..bdef010422 100644 --- a/cmake/SundialsSetupFortran.cmake +++ b/cmake/SundialsSetupFortran.cmake @@ -39,7 +39,8 @@ if(BUILD_FORTRAN_MODULE_INTERFACE) file(MAKE_DIRECTORY ${F2003Test_DIR}) # Create a CMakeLists.txt file - file(WRITE ${F2003Test_DIR}/CMakeLists.txt + file( + WRITE ${F2003Test_DIR}/CMakeLists.txt "CMAKE_MINIMUM_REQUIRED(VERSION ${CMAKE_VERSION})\n" "PROJECT(ftest Fortran)\n" "SET(CMAKE_VERBOSE_MAKEFILE ON)\n" @@ -54,29 +55,42 @@ if(BUILD_FORTRAN_MODULE_INTERFACE) # Create a Fortran source file which tries to use iso_c_binding file(WRITE ${F2003Test_DIR}/ftest.f90 - "program main\n" - "use, intrinsic :: iso_c_binding\n" - "end program main\n") + "program main\n" "use, intrinsic :: iso_c_binding\n" + "end program main\n") # Attempt compile the executable - try_compile(FTEST_OK ${F2003Test_DIR} ${F2003Test_DIR} - ftest OUTPUT_VARIABLE COMPILE_OUTPUT) + try_compile( + FTEST_OK ${F2003Test_DIR} + ${F2003Test_DIR} ftest + OUTPUT_VARIABLE COMPILE_OUTPUT) - # To ensure we do not use stuff from the previous attempts, - # we must remove the CMakeFiles directory. + # To ensure we do not use stuff from the previous attempts, we must remove + # the CMakeFiles directory. file(REMOVE_RECURSE ${F2003Test_DIR}/CMakeFiles) if(FTEST_OK) - message(STATUS "Checking whether ${CMAKE_Fortran_COMPILER} supports F2003 -- yes") - set(F2003_FOUND TRUE CACHE BOOL "${CMAKE_Fortran_COMPILER} supports F2003" FORCE) + message( + STATUS + "Checking whether ${CMAKE_Fortran_COMPILER} supports F2003 -- yes") + set(F2003_FOUND + TRUE + CACHE BOOL "${CMAKE_Fortran_COMPILER} supports F2003" FORCE) else() - message(STATUS "Checking whether ${CMAKE_Fortran_COMPILER} supports F2003 -- no") + message( + STATUS "Checking whether ${CMAKE_Fortran_COMPILER} supports F2003 -- no" + ) message(STATUS "Check output:") message("${COMPILE_OUTPUT}") - message(FATAL_ERROR "BUILD_FORTRAN_MODULE_INTERFACE is set to ON, but the CMAKE_Fortran_COMPILER does not support F2003") + message( + FATAL_ERROR + "BUILD_FORTRAN_MODULE_INTERFACE is set to ON, but the CMAKE_Fortran_COMPILER does not support F2003" + ) endif() else() - message(STATUS "Skipped F2003 tests, assuming ${CMAKE_Fortran_COMPILER} supports the f2003 standard. To rerun the F2003 tests, set F2003_FOUND to FALSE.") + message( + STATUS + "Skipped F2003 tests, assuming ${CMAKE_Fortran_COMPILER} supports the f2003 standard. To rerun the F2003 tests, set F2003_FOUND to FALSE." + ) endif() endif() diff --git a/cmake/SundialsSetupHIP.cmake b/cmake/SundialsSetupHIP.cmake index bc5a6c3776..709e335d55 100644 --- a/cmake/SundialsSetupHIP.cmake +++ b/cmake/SundialsSetupHIP.cmake @@ -16,25 +16,37 @@ if(NOT DEFINED ROCM_PATH) if(NOT DEFINED ENV{ROCM_PATH}) - set(ROCM_PATH "/opt/rocm/" CACHE PATH "Path to which ROCm has been installed") + set(ROCM_PATH + "/opt/rocm/" + CACHE PATH "Path to which ROCm has been installed") else() - set(ROCM_PATH "$ENV{ROCM_PATH}" CACHE PATH "Path to which ROCm has been installed") + set(ROCM_PATH + "$ENV{ROCM_PATH}" + CACHE PATH "Path to which ROCm has been installed") endif() endif() if(NOT DEFINED HIP_PATH) if(NOT DEFINED ENV{HIP_PATH}) - set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed") + set(HIP_PATH + "/opt/rocm/hip" + CACHE PATH "Path to which HIP has been installed") else() - set(HIP_PATH "$ENV{HIP_PATH}" CACHE PATH "Path to which HIP has been installed") + set(HIP_PATH + "$ENV{HIP_PATH}" + CACHE PATH "Path to which HIP has been installed") endif() endif() if(NOT DEFINED HIP_PLATFORM) if(NOT DEFINED ENV{HIP_PLATFORM}) - set(HIP_PLATFORM "amd" CACHE STRING "HIP platform (amd, nvidia)") + set(HIP_PLATFORM + "amd" + CACHE STRING "HIP platform (amd, nvidia)") else() - set(HIP_PLATFORM "$ENV{HIP_PLATFORM}" CACHE STRING "HIP platform (amd, nvidia)") + set(HIP_PLATFORM + "$ENV{HIP_PLATFORM}" + CACHE STRING "HIP platform (amd, nvidia)") endif() endif() @@ -46,7 +58,8 @@ set(CMAKE_PREFIX_PATH "${ROCM_PATH};${HIP_PATH}") find_package(HIP REQUIRED) if("${HIP_COMPILER}" STREQUAL "hcc") - message(FATAL_ERROR "Deprecated HCC compiler is not supported" "Please update ROCm") + message(FATAL_ERROR "Deprecated HCC compiler is not supported" + "Please update ROCm") endif() message(STATUS "HIP version: ${HIP_VERSION}") diff --git a/cmake/SundialsSetupTPLs.cmake b/cmake/SundialsSetupTPLs.cmake index fe57c20a7c..d23bb95503 100644 --- a/cmake/SundialsSetupTPLs.cmake +++ b/cmake/SundialsSetupTPLs.cmake @@ -15,8 +15,8 @@ # --------------------------------------------------------------- # --------------------------------------------------------------- -# Setup MPI, OpenMP, and OpenMP offload first as other TPLs may -# need targets or variables corresponding to these TPLs. +# Setup MPI, OpenMP, and OpenMP offload first as other TPLs may need targets or +# variables corresponding to these TPLs. # --------------------------------------------------------------- # --------------------------------------------------------------- @@ -77,7 +77,6 @@ if(ENABLE_GINKGO) list(APPEND SUNDIALS_TPL_LIST "GINKGO") endif() - # --------------------------------------------------------------- # Find (and test) the hypre libraries # --------------------------------------------------------------- diff --git a/cmake/SundialsSetupTesting.cmake b/cmake/SundialsSetupTesting.cmake index 11a445900d..b261ffe385 100644 --- a/cmake/SundialsSetupTesting.cmake +++ b/cmake/SundialsSetupTesting.cmake @@ -18,17 +18,25 @@ include(CTest) # Check if development tests are enabled -if (SUNDIALS_TEST_DEVTESTS OR BUILD_BENCHMARKS) +if(SUNDIALS_TEST_DEVTESTS OR BUILD_BENCHMARKS) # Python is needed to use the test runner find_package(Python3 REQUIRED) # look for the testRunner script in the test directory - find_program(TESTRUNNER testRunner PATHS test NO_DEFAULT_PATH) + find_program( + TESTRUNNER testRunner + PATHS test + NO_DEFAULT_PATH) if(NOT TESTRUNNER) - message(FATAL_ERROR "Could not locate testRunner. Set SUNDIALS_TEST_DEVTESTS=OFF or BUILD_BENCHMARKS=OFF to continue.") + message( + FATAL_ERROR + "Could not locate testRunner. Set SUNDIALS_TEST_DEVTESTS=OFF or BUILD_BENCHMARKS=OFF to continue." + ) endif() message(STATUS "Found testRunner: ${TESTRUNNER}") - set(TESTRUNNER ${TESTRUNNER} CACHE INTERNAL "") + set(TESTRUNNER + ${TESTRUNNER} + CACHE INTERNAL "") endif() @@ -46,7 +54,9 @@ if(SUNDIALS_TEST_DEVTESTS) # If a non-default output directory was provided make sure it exists if(SUNDIALS_TEST_OUTPUT_DIR) - message(STATUS "Using non-default test output directory: ${SUNDIALS_TEST_OUTPUT_DIR}") + message( + STATUS + "Using non-default test output directory: ${SUNDIALS_TEST_OUTPUT_DIR}") if(NOT EXISTS ${SUNDIALS_TEST_OUTPUT_DIR}) file(MAKE_DIRECTORY ${SUNDIALS_TEST_OUTPUT_DIR}) endif() @@ -54,7 +64,9 @@ if(SUNDIALS_TEST_DEVTESTS) # If a non-default answer directory was provided make sure it exists if(SUNDIALS_TEST_ANSWER_DIR) - message(STATUS "Using non-default test answer directory: ${SUNDIALS_TEST_ANSWER_DIR}") + message( + STATUS + "Using non-default test answer directory: ${SUNDIALS_TEST_ANSWER_DIR}") if(NOT EXISTS ${SUNDIALS_TEST_ANSWER_DIR}) message(FATAL_ERROR "SUNDIALS_TEST_ANSWER_DIR does not exist!") endif() @@ -62,7 +74,10 @@ if(SUNDIALS_TEST_DEVTESTS) # If a non-default caliper output directory was provided make sure it exists if(SUNDIALS_CALIPER_OUTPUT_DIR) - message(STATUS "Using non-default caliper output directory: ${SUNDIALS_CALIPER_OUTPUT_DIR}") + message( + STATUS + "Using non-default caliper output directory: ${SUNDIALS_CALIPER_OUTPUT_DIR}" + ) if(NOT EXISTS ${SUNDIALS_CALIPER_OUTPUT_DIR}/Example/${JOB_ID}) file(MAKE_DIRECTORY ${SUNDIALS_CALIPER_OUTPUT_DIR}/Example/${JOB_ID}) endif() @@ -70,11 +85,16 @@ if(SUNDIALS_TEST_DEVTESTS) # Check if using non-default comparison precisions when testing if(SUNDIALS_TEST_FLOAT_PRECISION GREATER_EQUAL "0") - message(STATUS "Using non-default float precision: ${SUNDIALS_TEST_FLOAT_PRECISION}") + message( + STATUS + "Using non-default float precision: ${SUNDIALS_TEST_FLOAT_PRECISION}") endif() if(SUNDIALS_TEST_INTEGER_PRECISION GREATER_EQUAL "0") - message(STATUS "Using non-default integer precision: ${SUNDIALS_TEST_INTEGER_PRECISION}") + message( + STATUS + "Using non-default integer precision: ${SUNDIALS_TEST_INTEGER_PRECISION}" + ) endif() # @@ -85,32 +105,57 @@ if(SUNDIALS_TEST_DEVTESTS) if(NOT container_exe) find_program(container_exe podman) endif() - set(SUNDIALS_TEST_CONTAINER_EXE ${container_exe} CACHE PATH "Path to docker or podman" FORCE) + set(SUNDIALS_TEST_CONTAINER_EXE + ${container_exe} + CACHE PATH "Path to docker or podman" FORCE) endif() if(SUNDIALS_TEST_CONTAINER_EXE) - add_custom_target(setup_local_ci - ${CMAKE_COMMAND} -E cmake_echo_color --cyan - "Pulled SUNDIALS CI containers.") + add_custom_target(setup_local_ci ${CMAKE_COMMAND} -E cmake_echo_color + --cyan "Pulled SUNDIALS CI containers.") - add_custom_target(test_local_ci - ${CMAKE_COMMAND} -E cmake_echo_color --cyan - "All testing with SUNDIALS CI containers complete.") + add_custom_target( + test_local_ci ${CMAKE_COMMAND} -E cmake_echo_color --cyan + "All testing with SUNDIALS CI containers complete.") macro(add_local_ci_target index_size precision tag) string(TOLOWER "${precision}" precision_) set(container sundials-ci-int${index_size}-${precision_}) - set(container_exe_args run ${SUNDIALS_TEST_CONTAINER_RUN_EXTRA_ARGS} -t -d --name ${container} --cap-add SYS_PTRACE - -v ${CMAKE_SOURCE_DIR}:${SUNDIALS_TEST_CONTAINER_MNT} ghcr.io/llnl/${container}:${tag}) - add_custom_target(setup_local_ci_${index_size}_${precision_} + set(container_exe_args + run + ${SUNDIALS_TEST_CONTAINER_RUN_EXTRA_ARGS} + -t + -d + --name + ${container} + --cap-add + SYS_PTRACE + -v + ${CMAKE_SOURCE_DIR}:${SUNDIALS_TEST_CONTAINER_MNT} + ghcr.io/llnl/${container}:${tag}) + add_custom_target( + setup_local_ci_${index_size}_${precision_} COMMENT "Pulling SUNDIALS CI container ghcr.io/llnl/${container}:${tag}" COMMAND ${SUNDIALS_TEST_CONTAINER_EXE} ${container_exe_args}) - add_dependencies(setup_local_ci setup_local_ci_${index_size}_${precision_}) + add_dependencies(setup_local_ci + setup_local_ci_${index_size}_${precision_}) set(container_test_exe ./test_driver.sh) - set(container_test_exe_args --testtype CUSTOM --env env/docker.sh --tpls --sunrealtype ${precision_} --indexsize ${index_size}) - set(container_exe_args exec -w ${SUNDIALS_TEST_CONTAINER_MNT}/test ${container} ${container_test_exe} ${container_test_exe_args}) - add_custom_target(test_local_ci_${index_size}_${precision_} + set(container_test_exe_args + --testtype + CUSTOM + --env + env/docker.sh + --tpls + --sunrealtype + ${precision_} + --indexsize + ${index_size}) + set(container_exe_args + exec -w ${SUNDIALS_TEST_CONTAINER_MNT}/test ${container} + ${container_test_exe} ${container_test_exe_args}) + add_custom_target( + test_local_ci_${index_size}_${precision_} COMMENT "Running tests in CI container ${container}:${tag}" WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${SUNDIALS_TEST_CONTAINER_EXE} ${container_exe_args} @@ -136,11 +181,13 @@ if(SUNDIALS_TEST_UNITTESTS AND SUNDIALS_TEST_ENABLE_GTEST) FetchContent_Declare( googletest URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip - GIT_TAG v1.14.0 - ) + GIT_TAG v1.14.0) if(WIN32) - # For Windows: Prevent overriding the parent project's compiler/linker settings - set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + # For Windows: Prevent overriding the parent project's compiler/linker + # settings + set(gtest_force_shared_crt + ON + CACHE BOOL "" FORCE) endif() FetchContent_MakeAvailable(googletest) include(GoogleTest) @@ -164,13 +211,11 @@ if(EXAMPLES_INSTALL) endif() # Create test_install and test_install_all targets - add_custom_target(test_install - ${CMAKE_COMMAND} -E cmake_echo_color --cyan - "All installation tests complete.") + add_custom_target(test_install ${CMAKE_COMMAND} -E cmake_echo_color --cyan + "All installation tests complete.") - add_custom_target(test_install_all - ${CMAKE_COMMAND} -E cmake_echo_color --cyan - "All installation tests complete.") + add_custom_target(test_install_all ${CMAKE_COMMAND} -E cmake_echo_color + --cyan "All installation tests complete.") endif() @@ -180,8 +225,6 @@ if(BUILD_BENCHMARKS) message("SUNDIALS Benchmarking") # Create benchmark targets - add_custom_target(benchmark - ${CMAKE_COMMAND} -E cmake_echo_color --cyan - "All benchmarks complete." - ) + add_custom_target(benchmark ${CMAKE_COMMAND} -E cmake_echo_color --cyan + "All benchmarks complete.") endif() diff --git a/cmake/SundialsTPLOptions.cmake b/cmake/SundialsTPLOptions.cmake index 36e0dc5a1b..765dcb4120 100644 --- a/cmake/SundialsTPLOptions.cmake +++ b/cmake/SundialsTPLOptions.cmake @@ -30,12 +30,11 @@ sundials_option(ENABLE_OPENMP BOOL "Enable OpenMP support" OFF) sundials_option(ENABLE_OPENMP_DEVICE BOOL "Enable OpenMP device offloading support" OFF) -# Advanced option to skip OpenMP device offloading support check. -# This is needed for a specific compiler that doesn't correctly -# report its OpenMP spec date (with CMake >= 3.9). +# Advanced option to skip OpenMP device offloading support check. This is needed +# for a specific compiler that doesn't correctly report its OpenMP spec date +# (with CMake >= 3.9). sundials_option(OPENMP_DEVICE_WORKS BOOL - "Skip the OpenMP device offloading support check" OFF - ADVANCED) + "Skip the OpenMP device offloading support check" OFF ADVANCED) # --------------------------------------------------------------- # Enable Pthread support? @@ -47,10 +46,6 @@ sundials_option(ENABLE_PTHREAD BOOL "Enable Pthreads support" OFF) # ------------------------------------------------------------- sundials_option(ENABLE_CUDA BOOL "Enable CUDA support" OFF) -# CMake 3.18 adds this option. -sundials_option(CMAKE_CUDA_ARCHITECTURES STRING "Target CUDA architecture" "70" - DEPENDS_ON ENABLE_CUDA) - # ------------------------------------------------------------- # Enable HIP support? # ------------------------------------------------------------- @@ -61,130 +56,148 @@ sundials_option(ENABLE_HIP BOOL "Enable HIP support" OFF) # ------------------------------------------------------------- sundials_option(ENABLE_SYCL BOOL "Enable SYCL support" OFF) -sundials_option(SUNDIALS_SYCL_2020_UNSUPPORTED BOOL - "Disable the use of some SYCL 2020 features in SUNDIALS libraries and examples" OFF - DEPENDS_ON ENABLE_SYCL - ADVANCED) +sundials_option( + SUNDIALS_SYCL_2020_UNSUPPORTED + BOOL + "Disable the use of some SYCL 2020 features in SUNDIALS libraries and examples" + OFF + DEPENDS_ON ENABLE_SYCL + ADVANCED) # --------------------------------------------------------------- # Enable LAPACK support? # --------------------------------------------------------------- sundials_option(ENABLE_LAPACK BOOL "Enable Lapack support" OFF) -sundials_option(LAPACK_LIBRARIES STRING "Lapack and Blas libraries" "${LAPACK_LIBRARIES}" - DEPENDS_ON ENABLE_LAPACK) +sundials_option(LAPACK_LIBRARIES STRING "Lapack and Blas libraries" + "${LAPACK_LIBRARIES}") -sundials_option(LAPACK_WORKS BOOL "Set to ON to force CMake to accept a given LAPACK configuration" OFF - DEPENDS_ON ENABLE_LAPACK - ADVANCED) +sundials_option( + LAPACK_WORKS BOOL + "Set to ON to force CMake to accept a given LAPACK configuration" OFF + ADVANCED) # --------------------------------------------------------------- # Enable Ginkgo support? # --------------------------------------------------------------- sundials_option(ENABLE_GINKGO BOOL "Enable Ginkgo support" OFF) -sundials_option(Ginkgo_DIR PATH "Path to the root of a Ginkgo installation" "${Ginkgo_DIR}" - DEPENDS_ON ENABLE_GINKGO) +sundials_option(Ginkgo_DIR PATH "Path to the root of a Ginkgo installation" + "${Ginkgo_DIR}") -sundials_option(SUNDIALS_GINKGO_BACKENDS STRING "Which Ginkgo backend(s) to build the SUNDIALS Ginkgo interfaces for (REF, OMP, CUDA, HIP, SYCL)" "REF;OMP" - DEPENDS_ON ENABLE_GINKGO) +sundials_option( + SUNDIALS_GINKGO_BACKENDS + STRING + "Which Ginkgo backend(s) to build the SUNDIALS Ginkgo interfaces for (REF, OMP, CUDA, HIP, SYCL)" + "REF;OMP" + DEPENDS_ON ENABLE_GINKGO) -sundials_option(GINKGO_WORKS BOOL "Set to ON to force CMake to accept a given Ginkgo configuration" OFF - DEPENDS_ON ENABLE_GINKGO - ADVANCED) +sundials_option( + GINKGO_WORKS BOOL + "Set to ON to force CMake to accept a given Ginkgo configuration" OFF + ADVANCED) # --------------------------------------------------------------- # Enable MAGMA support? # --------------------------------------------------------------- sundials_option(ENABLE_MAGMA BOOL "Enable MAGMA support" OFF) -sundials_option(MAGMA_DIR PATH "Path to the root of a MAGMA installation" "${MAGMA_DIR}" - DEPENDS_ON ENABLE_MAGMA) +sundials_option(MAGMA_DIR PATH "Path to the root of a MAGMA installation" + "${MAGMA_DIR}") -sundials_option(SUNDIALS_MAGMA_BACKENDS STRING "Which MAGMA backend to use under the SUNDIALS MAGMA interfaces (CUDA, HIP)" "CUDA" - OPTIONS "CUDA;HIP" - DEPENDS_ON ENABLE_MAGMA) +sundials_option( + SUNDIALS_MAGMA_BACKENDS STRING + "Which MAGMA backend to use under the SUNDIALS MAGMA interfaces (CUDA, HIP)" + "CUDA" + OPTIONS "CUDA;HIP" + DEPENDS_ON ENABLE_MAGMA) -sundials_option(MAGMA_WORKS BOOL "Set to ON to force CMake to accept a given MAGMA configuration" OFF - DEPENDS_ON ENABLE_MAGMA - ADVANCED) +sundials_option( + MAGMA_WORKS BOOL + "Set to ON to force CMake to accept a given MAGMA configuration" OFF ADVANCED) # --------------------------------------------------------------- # Enable SuperLU_DIST support? # --------------------------------------------------------------- sundials_option(ENABLE_SUPERLUDIST BOOL "Enable SuperLU_DIST support" OFF) -sundials_option(SUPERLUDIST_DIR PATH "Path to the root of the SuperLU_DIST installation" "${SUPERLUDIST_DIR}" - DEPENDS_ON ENABLE_SUPERLUDIST) +sundials_option( + SUPERLUDIST_DIR PATH "Path to the root of the SuperLU_DIST installation" + "${SUPERLUDIST_DIR}") -sundials_option(SUPERLUDIST_INCLUDE_DIRS PATH "SuperLU_DIST include directories" "${SUPERLUDIST_INCLUDE_DIRS}" - DEPENDS_ON ENABLE_SUPERLUDIST - ADVANCED) +sundials_option( + SUPERLUDIST_INCLUDE_DIRS PATH "SuperLU_DIST include directories" + "${SUPERLUDIST_INCLUDE_DIRS}" ADVANCED) -sundials_option(SUPERLUDIST_LIBRARIES STRING "Semi-colon separated list of libraries needed for SuperLU_DIST." "${SUPERLUDIST_LIBRARIES}" - DEPENDS_ON ENABLE_SUPERLUDIST - ADVANCED) +sundials_option( + SUPERLUDIST_LIBRARIES STRING + "Semi-colon separated list of libraries needed for SuperLU_DIST." + "${SUPERLUDIST_LIBRARIES}" ADVANCED) -sundials_option(SUPERLUDIST_OpenMP BOOL "Enable SUNDIALS support for SuperLU_DIST OpenMP on-node parallelism" OFF - DEPENDS_ON ENABLE_SUPERLUDIST) +sundials_option( + SUPERLUDIST_OpenMP BOOL + "Enable SUNDIALS support for SuperLU_DIST OpenMP on-node parallelism" OFF) -sundials_option(SUPERLUDIST_WORKS BOOL "Set to ON to force CMake to accept a given SuperLU_DIST configuration" OFF - DEPENDS_ON ENABLE_SUPERLUDIST - ADVANCED) +sundials_option( + SUPERLUDIST_WORKS BOOL + "Set to ON to force CMake to accept a given SuperLU_DIST configuration" OFF + ADVANCED) # --------------------------------------------------------------- # Enable SuperLU_MT support? # --------------------------------------------------------------- sundials_option(ENABLE_SUPERLUMT BOOL "Enable SuperLU_MT support" OFF) -sundials_option(SUPERLUMT_INCLUDE_DIR PATH "SuperLU_MT include directory" "${SUPERLUMT_INCLUDE_DIR}" - DEPENDS_ON ENABLE_SUPERLUMT) +sundials_option(SUPERLUMT_INCLUDE_DIR PATH "SuperLU_MT include directory" + "${SUPERLUMT_INCLUDE_DIR}") -sundials_option(SUPERLUMT_LIBRARY_DIR PATH "SuperLU_MT library directory" "${SUPERLUMT_LIBRARY_DIR}" - DEPENDS_ON ENABLE_SUPERLUMT) +sundials_option(SUPERLUMT_LIBRARY_DIR PATH "SuperLU_MT library directory" + "${SUPERLUMT_LIBRARY_DIR}") -sundials_option(SUPERLUMT_LIBRARIES STRING "Semi-colon separated list of additional libraries needed for SuperLU_MT." "${SUPERLUMT_LIBRARIES}" - DEPENDS_ON ENABLE_SUPERLUMT) +sundials_option( + SUPERLUMT_LIBRARIES STRING + "Semi-colon separated list of additional libraries needed for SuperLU_MT." + "${SUPERLUMT_LIBRARIES}") -sundials_option(SUPERLUMT_THREAD_TYPE STRING "SuperLU_MT threading type: OPENMP or PTHREAD" "PTHREAD" - DEPENDS_ON ENABLE_SUPERLUMT) +sundials_option(SUPERLUMT_THREAD_TYPE STRING + "SuperLU_MT threading type: OPENMP or PTHREAD" "PTHREAD") -sundials_option(SUPERLUMT_WORKS BOOL "Set to ON to force CMake to accept a given SUPERLUMT configuration" OFF - DEPENDS_ON ENABLE_SUPERLUMT - ADVANCED) +sundials_option( + SUPERLUMT_WORKS BOOL + "Set to ON to force CMake to accept a given SUPERLUMT configuration" OFF + ADVANCED) # --------------------------------------------------------------- # Enable KLU support? # --------------------------------------------------------------- sundials_option(ENABLE_KLU BOOL "Enable KLU support" OFF) -sundials_option(KLU_INCLUDE_DIR PATH "KLU include directory" "${KLU_INCLUDE_DIR}" - DEPENDS_ON ENABLE_KLU) +sundials_option(KLU_INCLUDE_DIR PATH "KLU include directory" + "${KLU_INCLUDE_DIR}") -sundials_option(KLU_LIBRARY_DIR PATH "KLU library directory" "${KLU_LIBRARY_DIR}" - DEPENDS_ON ENABLE_KLU) +sundials_option(KLU_LIBRARY_DIR PATH "KLU library directory" + "${KLU_LIBRARY_DIR}") -sundials_option(KLU_WORKS BOOL "Set to ON to force CMake to accept a given KLU configuration" OFF - DEPENDS_ON ENABLE_KLU - ADVANCED) +sundials_option( + KLU_WORKS BOOL "Set to ON to force CMake to accept a given KLU configuration" + OFF ADVANCED) # --------------------------------------------------------------- # Enable hypre support? # --------------------------------------------------------------- sundials_option(ENABLE_HYPRE BOOL "Enable hypre support" OFF) -sundials_option(HYPRE_DIR PATH "Path to hypre installation" "${HYPRE_DIR}" - DEPENDS_ON ENABLE_HYPRE) +sundials_option(HYPRE_DIR PATH "Path to hypre installation" "${HYPRE_DIR}") -sundials_option(HYPRE_INCLUDE_DIR PATH "HYPRE include directory" "${HYPRE_INCLUDE_DIR}" - DEPENDS_ON ENABLE_HYPRE) +sundials_option(HYPRE_INCLUDE_DIR PATH "HYPRE include directory" + "${HYPRE_INCLUDE_DIR}") -sundials_option(HYPRE_LIBRARY_DIR PATH "HYPRE library directory" "${HYPRE_LIBRARY_DIR}" - DEPENDS_ON ENABLE_HYPRE) +sundials_option(HYPRE_LIBRARY_DIR PATH "HYPRE library directory" + "${HYPRE_LIBRARY_DIR}") -sundials_option(HYPRE_WORKS BOOL "Set to ON to force CMake to accept a given hypre configuration" OFF - DEPENDS_ON ENABLE_HYPRE - ADVANCED) +sundials_option( + HYPRE_WORKS BOOL + "Set to ON to force CMake to accept a given hypre configuration" OFF ADVANCED) # --------------------------------------------------------------- # Enable PETSc support? @@ -192,72 +205,46 @@ sundials_option(HYPRE_WORKS BOOL "Set to ON to force CMake to accept a given hyp sundials_option(ENABLE_PETSC BOOL "Enable PETSc support" OFF) -sundials_option(PETSC_DIR PATH "Path to the root of a PETSc installation" "${PETSC_DIR}" - DEPENDS_ON ENABLE_PETSC) +sundials_option(PETSC_DIR PATH "Path to the root of a PETSc installation" + "${PETSC_DIR}") -sundials_option(PETSC_ARCH STRING "PETSc architecture (optional)" "${PETSC_ARCH}" - DEPENDS_ON ENABLE_PETSC) +sundials_option(PETSC_ARCH STRING "PETSc architecture (optional)" + "${PETSC_ARCH}") -sundials_option(PETSC_LIBRARIES STRING "Semi-colon separated list of PETSc link libraries" "${PETSC_LIBRARIES}" - DEPENDS_ON ENABLE_PETSC - ADVANCED) +sundials_option( + PETSC_LIBRARIES STRING "Semi-colon separated list of PETSc link libraries" + "${PETSC_LIBRARIES}" ADVANCED) -sundials_option(PETSC_INCLUDES STRING "Semi-colon separated list of PETSc include directories" "${PETSC_INCLUDES}" - DEPENDS_ON ENABLE_PETSC - ADVANCED) +sundials_option( + PETSC_INCLUDES STRING + "Semi-colon separated list of PETSc include directories" "${PETSC_INCLUDES}" + ADVANCED) -sundials_option(PETSC_WORKS BOOL "Set to ON to force CMake to accept a given PETSc configuration" OFF - DEPENDS_ON ENABLE_PETSC - ADVANCED) +sundials_option( + PETSC_WORKS BOOL + "Set to ON to force CMake to accept a given PETSc configuration" OFF ADVANCED) # ------------------------------------------------------------- # Enable RAJA support? # ------------------------------------------------------------- sundials_option(ENABLE_RAJA BOOL "Enable RAJA support" OFF) -sundials_option(RAJA_DIR PATH "Path to root of RAJA installation" "${RAJA_DIR}" - DEPENDS_ON ENABLE_RAJA) +sundials_option(RAJA_DIR PATH "Path to root of RAJA installation" "${RAJA_DIR}") -sundials_option(SUNDIALS_RAJA_BACKENDS STRING "Which RAJA backend under the SUNDIALS RAJA interfaces (CUDA, HIP, SYCL)" "CUDA" - OPTIONS "CUDA;HIP;SYCL" - DEPENDS_ON ENABLE_RAJA) +sundials_option( + SUNDIALS_RAJA_BACKENDS STRING + "Which RAJA backend under the SUNDIALS RAJA interfaces (CUDA, HIP, SYCL)" + "CUDA" + OPTIONS "CUDA;HIP;SYCL" + DEPENDS_ON ENABLE_RAJA) # --------------------------------------------------------------- # Enable Trilinos support? # --------------------------------------------------------------- sundials_option(ENABLE_TRILINOS BOOL "Enable Trilinos support" OFF) -sundials_option(Trilinos_DIR PATH "Path to root of Trilinos installation" "${Trilinos_DIR}" - DEPENDS_ON ENABLE_TRILINOS) - -sundials_option(Trilinos_INTERFACE_CXX_COMPILER STRING - "C++ compiler for Trilinos interface" "${Trilinos_CXX_COMPILER}" - DEPENDS_ON ENABLE_TRILINOS - ADVANCED) - -sundials_option(Trilinos_INTERFACE_C_COMPILER STRING - "C compiler for Trilinos interface" "${Trilinos_C_COMPILER}" - DEPENDS_ON ENABLE_TRILINOS - ADVANCED) - -sundials_option(Trilinos_INTERFACE_CXX_COMPILER_FLAGS STRING - "C++ compiler flags for Trilinos interface" "${Trilinos_CXX_COMPILER_FLAGS}" - DEPENDS_ON ENABLE_TRILINOS - ADVANCED) - -sundials_option(Trilinos_INTERFACE_C_COMPILER_FLAGS STRING - "C compiler flags for Trilinos interface" "${Trilinos_C_COMPILER_FLAGS}" - DEPENDS_ON ENABLE_TRILINOS - ADVANCED) - -sundials_option(Trilinos_INTERFACE_MPIEXEC STRING - "MPI executable for Trilinos interface" "${Trilinos_MPI_EXEC}" - DEPENDS_ON ENABLE_TRILINOS - ADVANCED) - -sundials_option(Trilinos_WORKS BOOL "Set to ON to force CMake to accept a given Trilinos configuration" OFF - DEPENDS_ON ENABLE_TRILINOS - ADVANCED) +sundials_option(Trilinos_DIR PATH "Path to root of Trilinos installation" + "${Trilinos_DIR}" DEPENDS_ON ENABLE_TRILINOS) # --------------------------------------------------------------- # Enable XBraid support? @@ -265,20 +252,22 @@ sundials_option(Trilinos_WORKS BOOL "Set to ON to force CMake to accept a given sundials_option(ENABLE_XBRAID BOOL "Enable XBraid support" OFF) -sundials_option(XBRAID_DIR PATH "Path to the root of an XBraid installation" "${XBRAID_DIR}" - DEPENDS_ON ENABLE_XBRAID) +sundials_option(XBRAID_DIR PATH "Path to the root of an XBraid installation" + "${XBRAID_DIR}") -sundials_option(XBRAID_LIBRARIES STRING "Semi-colon separated list of XBraid link libraries" "${XBRAID_LIBRARIES}" - DEPENDS_ON ENABLE_XBRAID - ADVANCED) +sundials_option( + XBRAID_LIBRARIES STRING "Semi-colon separated list of XBraid link libraries" + "${XBRAID_LIBRARIES}" ADVANCED) -sundials_option(XBRAID_INCLUDES STRING "Semi-colon separated list of XBraid include directories" "${XBRAID_INCLUDES}" - DEPENDS_ON ENABLE_XBRAID - ADVANCED) +sundials_option( + XBRAID_INCLUDES STRING + "Semi-colon separated list of XBraid include directories" + "${XBRAID_INCLUDES}" ADVANCED) -sundials_option(XBRAID_WORKS BOOL "Set to ON to force CMake to accept a given XBraid configuration" OFF - DEPENDS_ON ENABLE_XBRAID - ADVANCED) +sundials_option( + XBRAID_WORKS BOOL + "Set to ON to force CMake to accept a given XBraid configuration" OFF + ADVANCED) # ------------------------------------------------------------- # Enable oneMKL support? @@ -286,22 +275,25 @@ sundials_option(XBRAID_WORKS BOOL "Set to ON to force CMake to accept a given XB sundials_option(ENABLE_ONEMKL BOOL "Enable oneMKL support" OFF) -sundials_option(ONEMKL_DIR PATH "Path to root of oneMKL installation" "${ONEMKL_DIR}" - DEPENDS_ON ENABLE_ONEMKL) +sundials_option(ONEMKL_DIR PATH "Path to root of oneMKL installation" + "${ONEMKL_DIR}") -sundials_option(ONEMKL_WORKS BOOL "Set to ON to force CMake to accept a given oneMKL configuration" OFF - DEPENDS_ON ENABLE_ONEMKL - ADVANCED) +sundials_option( + ONEMKL_WORKS BOOL + "Set to ON to force CMake to accept a given oneMKL configuration" OFF + ADVANCED) -sundials_option(SUNDIALS_ONEMKL_USE_GETRF_LOOP BOOL - "Replace batched getrf call with loop over getrf" OFF - DEPENDS_ON ENABLE_ONEMKL - ADVANCED) +sundials_option( + SUNDIALS_ONEMKL_USE_GETRF_LOOP BOOL + "Replace batched getrf call with loop over getrf" OFF + DEPENDS_ON ENABLE_ONEMKL + ADVANCED) -sundials_option(SUNDIALS_ONEMKL_USE_GETRS_LOOP BOOL - "Replace batched getrs call with loop over getrs" OFF - DEPENDS_ON ENABLE_ONEMKL - ADVANCED) +sundials_option( + SUNDIALS_ONEMKL_USE_GETRS_LOOP BOOL + "Replace batched getrs call with loop over getrs" OFF + DEPENDS_ON ENABLE_ONEMKL + ADVANCED) # --------------------------------------------------------------- # Enable Caliper support? @@ -310,20 +302,23 @@ sundials_option(SUNDIALS_ONEMKL_USE_GETRS_LOOP BOOL sundials_option(ENABLE_CALIPER BOOL "Enable CALIPER support" OFF DEPENDS_ON SUNDIALS_BUILD_WITH_PROFILING) -sundials_option(CALIPER_DIR PATH "Path to the root of an CALIPER installation" "${CALIPER_DIR}" - DEPENDS_ON ENABLE_CALIPER) +sundials_option(CALIPER_DIR PATH "Path to the root of an CALIPER installation" + "${CALIPER_DIR}") -sundials_option(CALIPER_WORKS BOOL "Set to ON to force CMake to accept a given CALIPER configuration" OFF - DEPENDS_ON ENABLE_CALIPER - ADVANCED) +sundials_option( + CALIPER_WORKS BOOL + "Set to ON to force CMake to accept a given CALIPER configuration" OFF + ADVANCED) # --------------------------------------------------------------- # Enable Adiak support? # --------------------------------------------------------------- -sundials_option(ENABLE_ADIAK BOOL "Enable Adiak support" OFF DEPENDS_ON SUNDIALS_BUILD_WITH_PROFILING) +sundials_option(ENABLE_ADIAK BOOL "Enable Adiak support" OFF + DEPENDS_ON SUNDIALS_BUILD_WITH_PROFILING) -sundials_option(adiak_DIR PATH "Path to the root of an Adiak installation" "${ADIAK_DIR}" DEPENDS_ON ENABLE_ADIAK) +sundials_option(adiak_DIR PATH "Path to the root of an Adiak installation" + "${ADIAK_DIR}") # --------------------------------------------------------------- # Enable Kokkos support? @@ -331,11 +326,13 @@ sundials_option(adiak_DIR PATH "Path to the root of an Adiak installation" "${AD sundials_option(ENABLE_KOKKOS BOOL "Enable Kokkos support" OFF) -sundials_option(Kokkos_DIR PATH "Path to the root of a Kokkos installation" "${Kokkos_DIR}") +sundials_option(Kokkos_DIR PATH "Path to the root of a Kokkos installation" + "${Kokkos_DIR}") -sundials_option(KOKKOS_WORKS BOOL "Set to ON to force CMake to accept a given Kokkos configuration" OFF - DEPENDS_ON ENABLE_KOKKOS - ADVANCED) +sundials_option( + KOKKOS_WORKS BOOL + "Set to ON to force CMake to accept a given Kokkos configuration" OFF + ADVANCED) # --------------------------------------------------------------- # Enable Kokkos Kernels support? @@ -343,8 +340,11 @@ sundials_option(KOKKOS_WORKS BOOL "Set to ON to force CMake to accept a given Ko sundials_option(ENABLE_KOKKOS_KERNELS BOOL "Enable Kokkos Kernels support" OFF) -sundials_option(KokkosKernels_DIR PATH "Path to the root of a Kokkos Kernels installation" "${KokkosKernels_DIR}") +sundials_option( + KokkosKernels_DIR PATH "Path to the root of a Kokkos Kernels installation" + "${KokkosKernels_DIR}") -sundials_option(KOKKOS_KERNELS_WORKS BOOL "Set to ON to force CMake to accept a given Kokkos configuration" OFF - DEPENDS_ON ENABLE_KOKKOS ENABLE_KOKKOS_KERNELS - ADVANCED) +sundials_option( + KOKKOS_KERNELS_WORKS BOOL + "Set to ON to force CMake to accept a given Kokkos configuration" OFF + ADVANCED) diff --git a/cmake/macros/SundialsAddBenchmark.cmake b/cmake/macros/SundialsAddBenchmark.cmake index f3b3aec4e6..f5352f2d29 100644 --- a/cmake/macros/SundialsAddBenchmark.cmake +++ b/cmake/macros/SundialsAddBenchmark.cmake @@ -18,37 +18,41 @@ macro(sundials_add_benchmark NAME EXECUTABLE BASE_BENCHMARK_NAME) # Define single value parameters the macro takes in to set up the test runner # - # NUM_CORES = number of cores (GPU count or CPU count) to run on/number of resource sets - # BENCHMARK_ARGS = arguments to pass to the executable - # IDENTIFIER = suffix to append to end of benchmark name + # NUM_CORES = number of cores (GPU count or CPU count) to run + # on/number of resource sets BENCHMARK_ARGS = arguments to pass to the + # executable IDENTIFIER = suffix to append to end of benchmark name set(oneValueArgs NUM_CORES BENCHMARK_ARGS IDENTIFIER) # TEST_RUNNER_ARGS = command line arguments to pass to the test executable - set(multiValueArgs TEST_RUNNER_ARGS ) + set(multiValueArgs TEST_RUNNER_ARGS) # ENABLE_GPU = indicate this benchmark should be run with GPUs set(options ENABLE_GPU) - cmake_parse_arguments(sundials_add_benchmark - "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments(sundials_add_benchmark "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN}) # set the target name if(sundials_add_benchmark_IDENTIFIER) set(TARGET_NAME ${NAME}_${sundials_add_benchmark_IDENTIFIER}) else() if(sundials_add_benchmark_BENCHMARK_ARGS) - string(REPLACE " " "_" TEST_SUFFIX "${sundials_add_benchmark_BENCHMARK_ARGS}") + string(REPLACE " " "_" TEST_SUFFIX + "${sundials_add_benchmark_BENCHMARK_ARGS}") set(TARGET_NAME ${NAME}_${TEST_SUFFIX}) else() set(TARGET_NAME ${NAME}_run) endif() endif() - # Create default benchmark caliper output directory if custom directory is not defined + # Create default benchmark caliper output directory if custom directory is not + # defined if(SUNDIALS_CALIPER_OUTPUT_DIR) - set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${SUNDIALS_CALIPER_OUTPUT_DIR}/Benchmarking/${BASE_BENCHMARK_NAME}) + set(SUNDIALS_BENCHMARK_OUTPUT_DIR + ${SUNDIALS_CALIPER_OUTPUT_DIR}/Benchmarking/${BASE_BENCHMARK_NAME}) else() - set(SUNDIALS_BENCHMARK_OUTPUT_DIR ${PROJECT_BINARY_DIR}/Benchmarking/${BASE_BENCHMARK_NAME}) + set(SUNDIALS_BENCHMARK_OUTPUT_DIR + ${PROJECT_BINARY_DIR}/Benchmarking/${BASE_BENCHMARK_NAME}) endif() # make the caliper output directory if it doesn't exist @@ -63,22 +67,24 @@ macro(sundials_add_benchmark NAME EXECUTABLE BASE_BENCHMARK_NAME) # command line arguments for the test runner script set(TEST_RUNNER_ARGS - "--profile" - "--verbose" - "--executablename=$" - "--outputdir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/output" - "--calidir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}" - "--nodiff") + "--profile" "--verbose" "--executablename=$" + "--outputdir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/output" + "--calidir=${SUNDIALS_BENCHMARK_OUTPUT_DIR}/${TARGET_NAME}" "--nodiff") # incorporate scheduler arguments into test_runner if(SUNDIALS_SCHEDULER_COMMAND STREQUAL "flux run") set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_CORES}") - elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "jsrun" AND ${sundials_add_benchmark_ENABLE_GPU}) - set(SCHEDULER_STRING " --smpiargs=\\\"-gpu\\\" -n${sundials_add_benchmark_NUM_CORES} -a1 -c1 -g1") + elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "jsrun" + AND ${sundials_add_benchmark_ENABLE_GPU}) + set(SCHEDULER_STRING + " --smpiargs=\\\"-gpu\\\" -n${sundials_add_benchmark_NUM_CORES} -a1 -c1 -g1" + ) elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "jsrun") set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_CORES} -a1 -c1") elseif(SUNDIALS_SCHEDULER_COMMAND STREQUAL "srun") - set(SCHEDULER_STRING " -n${sundials_add_benchmark_NUM_CORES} --cpus-per-task=1 --ntasks-per-node=1") + set(SCHEDULER_STRING + " -n${sundials_add_benchmark_NUM_CORES} --cpus-per-task=1 --ntasks-per-node=1" + ) endif() string(REPLACE " " ";" SCHEDULER_ARGS "${SCHEDULER_STRING}") string(REPLACE " " ";" SCHEDULER_COMMAND_ARGS "${SUNDIALS_SCHEDULER_COMMAND}") @@ -87,8 +93,11 @@ macro(sundials_add_benchmark NAME EXECUTABLE BASE_BENCHMARK_NAME) set(RUN_COMMAND ${SCHEDULER_COMMAND_ARGS} ${SCHEDULER_ARGS}) list(APPEND TEST_RUNNER_ARGS "--runcommand=\"${RUN_COMMAND}\"") - list(APPEND TEST_RUNNER_ARGS "--runargs=${sundials_add_benchmark_BENCHMARK_ARGS}" "--testname=${TARGET_NAME}") - add_custom_target(${TARGET_NAME} + list(APPEND TEST_RUNNER_ARGS + "--runargs=${sundials_add_benchmark_BENCHMARK_ARGS}" + "--testname=${TARGET_NAME}") + add_custom_target( + ${TARGET_NAME} COMMENT "Running ${TARGET_NAME}" COMMAND ${PYTHON_EXECUTABLE} ${TESTRUNNER} ${TEST_RUNNER_ARGS}) add_dependencies(benchmark ${TARGET_NAME}) diff --git a/cmake/macros/SundialsAddExamplesGinkgo.cmake b/cmake/macros/SundialsAddExamplesGinkgo.cmake index 1e23dfdb58..c723f68f29 100644 --- a/cmake/macros/SundialsAddExamplesGinkgo.cmake +++ b/cmake/macros/SundialsAddExamplesGinkgo.cmake @@ -40,8 +40,8 @@ macro(sundials_add_examples_ginkgo EXAMPLES_VAR) set(multiValueArgs TARGETS BACKENDS) # Parse keyword arguments and options - cmake_parse_arguments(arg - "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments(arg "${options}" "${oneValueArgs}" "${multiValueArgs}" + ${ARGN}) foreach(example_tuple ${${EXAMPLES_VAR}}) foreach(backend ${arg_BACKENDS}) @@ -87,17 +87,13 @@ macro(sundials_add_examples_ginkgo EXAMPLES_VAR) target_compile_definitions(${example_target} PRIVATE USE_${backend}) # directories to include - target_include_directories(${example_target} - PRIVATE - "${PROJECT_SOURCE_DIR}/examples/utilities") + target_include_directories( + ${example_target} PRIVATE "${PROJECT_SOURCE_DIR}/examples/utilities") # libraries to link against - target_link_libraries(${example_target} - PRIVATE - ${arg_TARGETS} - sundials_${vector} - Ginkgo::ginkgo - ${EXTRA_LINK_LIBS}) + target_link_libraries( + ${example_target} PRIVATE ${arg_TARGETS} sundials_${vector} + Ginkgo::ginkgo ${EXTRA_LINK_LIBS}) endif() @@ -105,17 +101,20 @@ macro(sundials_add_examples_ginkgo EXAMPLES_VAR) if("${example_args}" STREQUAL "") set(test_name ${example_target}) else() - string(REGEX REPLACE " " "_" test_name ${example_target}_${example_args}) + string(REGEX REPLACE " " "_" test_name + ${example_target}_${example_args}) endif() # add example to regression tests if(${arg_UNIT_TEST}) - sundials_add_test(${test_name} ${example_target} + sundials_add_test( + ${test_name} ${example_target} EXAMPLE_TYPE ${example_type} TEST_ARGS ${example_args} NODIFF) else() - sundials_add_test(${test_name} ${example_target} + sundials_add_test( + ${test_name} ${example_target} EXAMPLE_TYPE ${example_type} TEST_ARGS ${example_args} ANSWER_DIR ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/cmake/macros/SundialsAddExecutable.cmake b/cmake/macros/SundialsAddExecutable.cmake index a2582a3485..00508c76b2 100644 --- a/cmake/macros/SundialsAddExecutable.cmake +++ b/cmake/macros/SundialsAddExecutable.cmake @@ -16,30 +16,26 @@ macro(sundials_add_nvector_benchmark NAME) - set(options ) - set(singleValueArgs ) - set(multiValueArgs SOURCES SUNDIALS_TARGETS LINK_LIBRARIES - INSTALL_SUBDIR) + set(options) + set(singleValueArgs) + set(multiValueArgs SOURCES SUNDIALS_TARGETS LINK_LIBRARIES INSTALL_SUBDIR) - cmake_parse_arguments(arg - "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments(arg "${options}" "${singleValueArgs}" + "${multiValueArgs}" ${ARGN}) set(BENCHMARKS_DIR ${PROJECT_SOURCE_DIR}/benchmarks) - add_executable(${NAME} - ${BENCHMARKS_DIR}/nvector/test_nvector_performance.c - ${arg_SOURCES}) + add_executable(${NAME} ${BENCHMARKS_DIR}/nvector/test_nvector_performance.c + ${arg_SOURCES}) set_target_properties(${NAME} PROPERTIES FOLDER "Benchmarks") - target_include_directories(${NAME} PRIVATE - ${BENCHMARKS_DIR}/nvector) + target_include_directories(${NAME} PRIVATE ${BENCHMARKS_DIR}/nvector) - target_link_libraries(${NAME} PRIVATE - ${arg_SUNDIALS_TARGETS} ${arg_LINK_LIBRARIES} -lm) + target_link_libraries(${NAME} PRIVATE ${arg_SUNDIALS_TARGETS} + ${arg_LINK_LIBRARIES} -lm) install(TARGETS ${NAME} - DESTINATION "${BENCHMARKS_INSTALL_PATH}/${arg_INSTALL_SUBDIR}") + DESTINATION "${BENCHMARKS_INSTALL_PATH}/${arg_INSTALL_SUBDIR}") endmacro(sundials_add_nvector_benchmark) - diff --git a/cmake/macros/SundialsAddLibrary.cmake b/cmake/macros/SundialsAddLibrary.cmake index 199f790e66..e3d0bb893d 100644 --- a/cmake/macros/SundialsAddLibrary.cmake +++ b/cmake/macros/SundialsAddLibrary.cmake @@ -15,45 +15,49 @@ # Wraps the add_library command for sundials specific needs. # --------------------------------------------------------------- - -# The macro: -# -# SUNDIALS_ADD_LIBRARY( -# SOURCES source1 source2 ... -# [HEADERS header1 header2 ...] -# [OBJECT_LIBRARIES objlib1 objlib2 ...] -# [LINK_LIBRARIES ... +# ~~~ +# sundials_add_library( +# SOURCES source1 source2 ... +# [HEADERS header1 header2 ...] +# [OBJECT_LIBRARIES objlib1 objlib2 ...] +# [LINK_LIBRARIES ... +# [ ...] ] +# [INCLUDE_DIRECTORIES ... +# [ ...] ] +# [COMPILE_DEFINITIONS ... +# [ ...] ] +# [COMPILE_OPTIONS ... +# [ ...] ] +# [COMPILE_FEATURES ... # [ ...] ] -# [INCLUDE_DIRECTORIES ... -# [ ...] ] -# [COMPILE_DEFINITIONS ... -# [ ...] ] -# [COMPILE_OPTIONS ... -# [ ...] ] -# [COMPILE_FEATURES ... -# [ ...] ] -# [PROPERTIES ... [ ...] ] -# [INCLUDE_SUBDIR] -# [OUTPUT_NAME name] -# [VERSION version] -# [SOVERSION version] -# [STATIC_ONLY | SHARED_ONLY] -# [OBJECT_LIB_ONLY]) +# [PROPERTIES ... [ ...] ] +# [INCLUDE_SUBDIR] +# [OUTPUT_NAME name] +# [VERSION version] +# [SOVERSION version] +# [STATIC_ONLY | SHARED_ONLY] +# [OBJECT_LIB_ONLY]) +# ~~~ # -# adds libraries to be built from the source files listed in the command +# Adds libraries to be built from the source files listed in the command # invocation. It is a convenient wrapper of the CMake add_library command that # is specific to our usage of add_library in SUNDIALS. # # By default, the macro uses the CMake add_library command to create the # targets: -# - ${_SHARED_LIB_SUFFIX} (will be a shared library) -# - ${_STATIC_LIB_SUFFIX} (will be a static library) -# - _obj${_SHARED_LIB_SUFFIX} (an object library that is used to -# create ${_SHARED_LIB_SUFFIX}) -# - _obj${_STATIC_LIB_SUFFIX} (an object library that is used to -# create ${_STATIC_LIB_SUFFIX}) -# - (an alias to the shared library, if enabled, otherwise an -# alias to the static library) +# +# * ${_SHARED_LIB_SUFFIX} (will be a shared library) +# +# * ${_STATIC_LIB_SUFFIX} (will be a static library) +# +# * _obj${_SHARED_LIB_SUFFIX} (an object library that is used to create +# ${_SHARED_LIB_SUFFIX}) +# +# * _obj${_STATIC_LIB_SUFFIX} (an object library that is used to create +# ${_STATIC_LIB_SUFFIX}) +# +# * (an alias to the shared library, if enabled, otherwise an alias to +# the static library) # # The SOURCES input is a list of source files used to create the library. # @@ -108,17 +112,25 @@ # # The option OBJECT_LIB_ONLY will cause the macro to only create the object # library targets. + macro(sundials_add_library target) set(options STATIC_ONLY SHARED_ONLY OBJECT_LIB_ONLY) set(oneValueArgs INCLUDE_SUBDIR OUTPUT_NAME VERSION SOVERSION) - set(multiValueArgs SOURCES HEADERS OBJECT_LIBRARIES LINK_LIBRARIES - INCLUDE_DIRECTORIES COMPILE_DEFINITIONS COMPILE_OPTIONS - COMPILE_FEATURES PROPERTIES) + set(multiValueArgs + SOURCES + HEADERS + OBJECT_LIBRARIES + LINK_LIBRARIES + INCLUDE_DIRECTORIES + COMPILE_DEFINITIONS + COMPILE_OPTIONS + COMPILE_FEATURES + PROPERTIES) # parse keyword arguments/options - cmake_parse_arguments(sundials_add_library - "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments(sundials_add_library "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN}) # library types to create set(_libtypes "") @@ -150,14 +162,15 @@ macro(sundials_add_library target) # -------------------------------------------------------------------------- # create the target for the object library - add_library(${obj_target} OBJECT ${sources} ${sundials_add_library_UNPARSED_ARGUMENTS}) + add_library(${obj_target} OBJECT ${sources} + ${sundials_add_library_UNPARSED_ARGUMENTS}) set_target_properties(${obj_target} PROPERTIES FOLDER "obj") # add all object libraries to object library if(sundials_add_library_OBJECT_LIBRARIES) target_link_libraries(${obj_target} - PRIVATE ${sundials_add_library_OBJECT_LIBRARIES}) + PRIVATE ${sundials_add_library_OBJECT_LIBRARIES}) endif() # add all link libraries to object library @@ -173,9 +186,11 @@ macro(sundials_add_library target) else() set(_all_libs ${sundials_add_library_LINK_LIBRARIES}) endif() - # Due to various issues in CMake, particularly https://gitlab.kitware.com/cmake/cmake/-/issues/25365, - # we create a fake custom target to enforce a build order. Without this, parallel builds - # might fail with an error about a missing '.mod' file when Fortran is enabled (see GitHub #410). + # Due to various issues in CMake, particularly + # https://gitlab.kitware.com/cmake/cmake/-/issues/25365, we create a fake + # custom target to enforce a build order. Without this, parallel builds + # might fail with an error about a missing '.mod' file when Fortran is + # enabled (see GitHub #410). set(_stripped_all_libs ${_all_libs}) list(FILTER _stripped_all_libs EXCLUDE REGEX "PUBLIC|INTERFACE|PRIVATE") foreach(_item ${_stripped_all_libs}) @@ -184,7 +199,8 @@ macro(sundials_add_library target) endif() endforeach() add_custom_target(fake_to_force_build_order_${obj_target}) - add_dependencies(fake_to_force_build_order_${obj_target} ${_stripped_all_libs}) + add_dependencies(fake_to_force_build_order_${obj_target} + ${_stripped_all_libs}) add_dependencies(${obj_target} fake_to_force_build_order_${obj_target}) target_link_libraries(${obj_target} ${_all_libs}) endif() @@ -194,20 +210,21 @@ macro(sundials_add_library target) target_link_libraries(${obj_target} PUBLIC caliper) endif() if(ENABLE_ADIAK) - target_link_libraries(${obj_target} PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) + target_link_libraries(${obj_target} PUBLIC adiak::adiak + ${CMAKE_DL_LIBS}) endif() endif() # add includes to object library - target_include_directories(${obj_target} - PUBLIC - $ - $ - $ - $ - ) + target_include_directories( + ${obj_target} + PUBLIC $ + $ + $ + $) if(sundials_add_library_INCLUDE_DIRECTORIES) - string(REPLACE "{{libtype}}" "${_libtype}" _includes "${sundials_add_library_INCLUDE_DIRECTORIES}") + string(REPLACE "{{libtype}}" "${_libtype}" _includes + "${sundials_add_library_INCLUDE_DIRECTORIES}") target_include_directories(${obj_target} ${_includes}) endif() @@ -220,25 +237,30 @@ macro(sundials_add_library target) # add all other compile definitions to object library if(sundials_add_library_COMPILE_DEFINITIONS) - target_compile_definitions(${obj_target} ${sundials_add_library_COMPILE_DEFINITIONS}) + target_compile_definitions(${obj_target} + ${sundials_add_library_COMPILE_DEFINITIONS}) endif() # add compile options to object library if(sundials_add_library_COMPILE_OPTIONS) - target_compile_options(${obj_target} ${sundials_add_library_COMPILE_OPTIONS}) + target_compile_options(${obj_target} + ${sundials_add_library_COMPILE_OPTIONS}) endif() # add compile features if(sundials_add_library_COMPILE_FEATURES) - target_compile_features(${obj_target} ${sundials_add_library_COMPILE_FEATURES}) + target_compile_features(${obj_target} + ${sundials_add_library_COMPILE_FEATURES}) endif() # object files going into shared libs need PIC code - set_target_properties(${obj_target} PROPERTIES POSITION_INDEPENDENT_CODE TRUE) + set_target_properties(${obj_target} PROPERTIES POSITION_INDEPENDENT_CODE + TRUE) # set any other properties if(sundials_add_library_PROPERTIES) - string(REPLACE "{{libtype}}" "${_libtype}" _properties "${sundials_add_library_PROPERTIES}") + string(REPLACE "{{libtype}}" "${_libtype}" _properties + "${sundials_add_library_PROPERTIES}") set_target_properties(${obj_target} PROPERTIES ${_properties}) endif() @@ -264,19 +286,24 @@ macro(sundials_add_library target) endforeach() endif() - add_library(${_actual_target_name} ${_libtype} ${_object_sources} ${sundials_add_library_UNPARSED_ARGUMENTS}) + add_library( + ${_actual_target_name} ${_libtype} ${_object_sources} + ${sundials_add_library_UNPARSED_ARGUMENTS}) set_target_properties(${_actual_target_name} PROPERTIES FOLDER "src") # add all link libraries if(SUNDIALS_MATH_LIBRARY) - target_link_libraries(${_actual_target_name} PRIVATE "${SUNDIALS_MATH_LIBRARY}") + target_link_libraries(${_actual_target_name} + PRIVATE "${SUNDIALS_MATH_LIBRARY}") endif() if(SUNDIALS_RT_LIBRARY) - target_link_libraries(${_actual_target_name} PRIVATE "${SUNDIALS_RT_LIBRARY}") + target_link_libraries(${_actual_target_name} + PRIVATE "${SUNDIALS_RT_LIBRARY}") endif() if(sundials_add_library_LINK_LIBRARIES) - target_link_libraries(${_actual_target_name} ${sundials_add_library_LINK_LIBRARIES}) + target_link_libraries(${_actual_target_name} + ${sundials_add_library_LINK_LIBRARIES}) endif() if(SUNDIALS_BUILD_WITH_PROFILING) @@ -284,90 +311,103 @@ macro(sundials_add_library target) target_link_libraries(${_actual_target_name} PUBLIC caliper) endif() if(ENABLE_ADIAK) - target_link_libraries(${_actual_target_name} PUBLIC adiak::adiak ${CMAKE_DL_LIBS}) + target_link_libraries(${_actual_target_name} PUBLIC adiak::adiak + ${CMAKE_DL_LIBS}) endif() endif() # add common includes + # # Building: public, config/export generated, and shared private headers + # # Installing: installed include directory - target_include_directories(${_actual_target_name} PUBLIC - $ - $ - $ - $ - $) + target_include_directories( + ${_actual_target_name} + PUBLIC $ + $ + $ + $ + $) # add all other includes if(sundials_add_library_INCLUDE_DIRECTORIES) - string(REPLACE "{{libtype}}" "${_libtype}" _includes "${sundials_add_library_INCLUDE_DIRECTORIES}") + string(REPLACE "{{libtype}}" "${_libtype}" _includes + "${sundials_add_library_INCLUDE_DIRECTORIES}") target_include_directories(${_actual_target_name} ${_includes}) endif() # add compile definitions for SUNDIALS_EXPORT if(${_libtype} MATCHES "STATIC") - target_compile_definitions(${_actual_target_name} PUBLIC SUNDIALS_STATIC_DEFINE) + target_compile_definitions(${_actual_target_name} + PUBLIC SUNDIALS_STATIC_DEFINE) else() target_compile_definitions(${obj_target} PRIVATE sundials_core_EXPORTS) endif() # add all other compile definitions if(sundials_add_library_COMPILE_DEFINITIONS) - target_compile_definitions(${_actual_target_name} ${sundials_add_library_COMPILE_DEFINITIONS}) + target_compile_definitions(${_actual_target_name} + ${sundials_add_library_COMPILE_DEFINITIONS}) endif() # add all compile options if(sundials_add_library_COMPILE_OPTIONS) - target_compile_options(${_actual_target_name} ${sundials_add_library_COMPILE_OPTIONS}) + target_compile_options(${_actual_target_name} + ${sundials_add_library_COMPILE_OPTIONS}) endif() # add compile features if(sundials_add_library_COMPILE_FEATURES) - target_compile_features(${_actual_target_name} ${sundials_add_library_COMPILE_FEATURES}) + target_compile_features(${_actual_target_name} + ${sundials_add_library_COMPILE_FEATURES}) endif() - # exported targets are in the SUNDIALS:: namespace, so we remove the sundials_ prefix from the exported name + # exported targets are in the SUNDIALS:: namespace, so we remove the + # sundials_ prefix from the exported name string(REPLACE "sundials_" "" _export_name "${_actual_target_name}") - set_target_properties(${_actual_target_name} PROPERTIES EXPORT_NAME ${_export_name}) + set_target_properties(${_actual_target_name} PROPERTIES EXPORT_NAME + ${_export_name}) - # create an alias to match the exported target name, this way another projects can use it with either find_package() or add_subdirectory() + # create an alias to match the exported target name, this way another + # projects can use it with either find_package() or add_subdirectory() add_library(SUNDIALS::${_export_name} ALIAS ${_actual_target_name}) # set the correct output name if(sundials_add_library_OUTPUT_NAME) - if((MSVC OR ("${CMAKE_C_SIMULATE_ID}" STREQUAL "MSVC")) AND ${_libtype} MATCHES "STATIC") - set_target_properties(${_actual_target_name} PROPERTIES - OUTPUT_NAME "${sundials_add_library_OUTPUT_NAME}_static" - CLEAN_DIRECT_OUTPUT 1 - ) + if((MSVC OR ("${CMAKE_C_SIMULATE_ID}" STREQUAL "MSVC")) + AND ${_libtype} MATCHES "STATIC") + set_target_properties( + ${_actual_target_name} + PROPERTIES OUTPUT_NAME "${sundials_add_library_OUTPUT_NAME}_static" + CLEAN_DIRECT_OUTPUT 1) else() - set_target_properties(${_actual_target_name} PROPERTIES - OUTPUT_NAME ${sundials_add_library_OUTPUT_NAME} - CLEAN_DIRECT_OUTPUT 1 - ) + set_target_properties( + ${_actual_target_name} + PROPERTIES OUTPUT_NAME ${sundials_add_library_OUTPUT_NAME} + CLEAN_DIRECT_OUTPUT 1) endif() else() - set_target_properties(${_actual_target_name} PROPERTIES - OUTPUT_NAME ${target} - CLEAN_DIRECT_OUTPUT 1 - ) + set_target_properties( + ${_actual_target_name} PROPERTIES OUTPUT_NAME ${target} + CLEAN_DIRECT_OUTPUT 1) endif() # set the library versions if(sundials_add_library_VERSION) - set_target_properties(${_actual_target_name} PROPERTIES - VERSION ${sundials_add_library_VERSION} - ) + set_target_properties( + ${_actual_target_name} PROPERTIES VERSION + ${sundials_add_library_VERSION}) endif() if(sundials_add_library_SOVERSION) - set_target_properties(${_actual_target_name} PROPERTIES - SOVERSION ${sundials_add_library_SOVERSION} - ) + set_target_properties( + ${_actual_target_name} PROPERTIES SOVERSION + ${sundials_add_library_SOVERSION}) endif() # set any other properties if(sundials_add_library_PROPERTIES) - string(REPLACE "{{libtype}}" "${_libtype}" _properties "${sundials_add_library_PROPERTIES}") + string(REPLACE "{{libtype}}" "${_libtype}" _properties + "${sundials_add_library_PROPERTIES}") set_target_properties(${_actual_target_name} PROPERTIES ${_properties}) endif() @@ -383,8 +423,10 @@ macro(sundials_add_library target) # -------------------------------------------------------------------------- if(sundials_add_library_HEADERS) - install(FILES ${sundials_add_library_HEADERS} - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${sundials_add_library_INCLUDE_SUBDIR}") + install( + FILES ${sundials_add_library_HEADERS} + DESTINATION + "${CMAKE_INSTALL_INCLUDEDIR}/${sundials_add_library_INCLUDE_SUBDIR}") endif() # -------------------------------------------------------------------------- @@ -395,21 +437,27 @@ macro(sundials_add_library target) add_library(${target}_obj ALIAS ${target}_obj${_SHARED_LIB_SUFFIX}) if(NOT sundials_add_library_OBJECT_LIB_ONLY) add_library(${target} ALIAS ${target}${_SHARED_LIB_SUFFIX}) - set(_SUNDIALS_ALIAS_TARGETS "${target}->${target}${_SHARED_LIB_SUFFIX};${_SUNDIALS_ALIAS_TARGETS}" CACHE INTERNAL "" FORCE) + set(_SUNDIALS_ALIAS_TARGETS + "${target}->${target}${_SHARED_LIB_SUFFIX};${_SUNDIALS_ALIAS_TARGETS}" + CACHE INTERNAL "" FORCE) # Namespaced alias for using build directory directly string(REPLACE "sundials_" "" _export_name "${target}") - add_library(SUNDIALS::${_export_name} ALIAS ${target}${_SHARED_LIB_SUFFIX}) + add_library(SUNDIALS::${_export_name} ALIAS + ${target}${_SHARED_LIB_SUFFIX}) endif() else() add_library(${target}_obj ALIAS ${target}_obj${_STATIC_LIB_SUFFIX}) if(NOT sundials_add_library_OBJECT_LIB_ONLY) add_library(${target} ALIAS ${target}${_STATIC_LIB_SUFFIX}) - set(_SUNDIALS_ALIAS_TARGETS "${target}->${target}${_STATIC_LIB_SUFFIX};${_SUNDIALS_ALIAS_TARGETS}" CACHE INTERNAL "" FORCE) + set(_SUNDIALS_ALIAS_TARGETS + "${target}->${target}${_STATIC_LIB_SUFFIX};${_SUNDIALS_ALIAS_TARGETS}" + CACHE INTERNAL "" FORCE) # Namespaced alias for using build directory directly string(REPLACE "sundials_" "" _export_name "${target}") - add_library(SUNDIALS::${_export_name} ALIAS ${target}${_STATIC_LIB_SUFFIX}) + add_library(SUNDIALS::${_export_name} ALIAS + ${target}${_STATIC_LIB_SUFFIX}) endif() endif() @@ -419,40 +467,45 @@ macro(sundials_add_library target) if(NOT sundials_add_library_OBJECT_LIB_ONLY) string(REPLACE "sundials_" "" _comp_name "${target}") - set(_SUNDIALS_INSTALLED_COMPONENTS "${_comp_name};${_SUNDIALS_INSTALLED_COMPONENTS}" CACHE INTERNAL "" FORCE) + set(_SUNDIALS_INSTALLED_COMPONENTS + "${_comp_name};${_SUNDIALS_INSTALLED_COMPONENTS}" + CACHE INTERNAL "" FORCE) endif() endmacro(sundials_add_library) - macro(sundials_add_f2003_library target) - set(options ) + set(options) set(oneValueArgs OUTPUT_NAME VERSION SOVERSION) - set(multiValueArgs SOURCES OBJECT_LIBRARIES LINK_LIBRARIES INCLUDE_DIRECTORIES - COMPILE_DEFINITIONS COMPILE_OPTIONS PROPERTIES) + set(multiValueArgs + SOURCES + OBJECT_LIBRARIES + LINK_LIBRARIES + INCLUDE_DIRECTORIES + COMPILE_DEFINITIONS + COMPILE_OPTIONS + PROPERTIES) # parse keyword arguments/options - cmake_parse_arguments(sundials_add_f2003_library - "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments(sundials_add_f2003_library "${options}" + "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) if(CMAKE_Fortran_MODULE_DIRECTORY) set(_includes - PUBLIC - $ - $ - ) - set(_properties PROPERTIES - Fortran_MODULE_DIRECTORY "${CMAKE_Fortran_MODULE_DIRECTORY}_{{libtype}}" + PUBLIC $ + $) + set(_properties + PROPERTIES Fortran_MODULE_DIRECTORY + "${CMAKE_Fortran_MODULE_DIRECTORY}_{{libtype}}" WINDOWS_EXPORT_ALL_SYMBOLS ON) else() set(_includes - PUBLIC - $ - $ - ) - set(_properties PROPERTIES - Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${target}.dir" + PUBLIC $ + $) + set(_properties + PROPERTIES Fortran_MODULE_DIRECTORY + "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${target}.dir" WINDOWS_EXPORT_ALL_SYMBOLS ON) endif() @@ -462,30 +515,27 @@ macro(sundials_add_f2003_library target) if(TARGET ${_clib_name}) set(_clib_target ${_clib_name}) else() - set(_clib_target ) + set(_clib_target) endif() - sundials_add_library(${target} + sundials_add_library( + ${target} SOURCES ${sundials_add_f2003_library_SOURCES} OBJECT_LIBRARIES ${sundials_add_f2003_library_OBJECT_LIBRARIES} - LINK_LIBRARIES - PUBLIC ${_clib_target} # depend on the c library - ${sundials_add_f2003_library_LINK_LIBRARIES} - INCLUDE_DIRECTORIES - ${sundials_add_f2003_library_INCLUDE_DIRECTORIES} - ${_includes} - COMPILE_DEFINITIONS ${sundials_add_f2003_library_COMPILE_DEFINITIONS} - PUBLIC "SUNDIALS_INT${SUNDIALS_INDEX_SIZE}_T" + LINK_LIBRARIES PUBLIC ${_clib_target} # depend on the c library + ${sundials_add_f2003_library_LINK_LIBRARIES} + INCLUDE_DIRECTORIES ${sundials_add_f2003_library_INCLUDE_DIRECTORIES} + ${_includes} + COMPILE_DEFINITIONS ${sundials_add_f2003_library_COMPILE_DEFINITIONS} PUBLIC + "SUNDIALS_INT${SUNDIALS_INDEX_SIZE}_T" COMPILE_OPTIONS ${sundials_add_f2003_library_COMPILE_OPTIONS} PROPERTIES ${sundials_add_f2003_library_PROPERTIES} ${_properties} OUTPUT_NAME ${sundials_add_f2003_library_OUTPUT_NAME} VERSION ${sundials_add_f2003_library_VERSION} SOVERSION ${sundials_add_f2003_library_SOVERSION} - ${sundials_add_f2003_library_UNPARSED_ARGUMENTS} - ) + ${sundials_add_f2003_library_UNPARSED_ARGUMENTS}) endmacro() - macro(append_static_suffix libs_in libs_out) set(${libs_out} "") foreach(_lib ${${libs_in}}) diff --git a/cmake/macros/SundialsAddTest.cmake b/cmake/macros/SundialsAddTest.cmake index 33eb8d7fa1..6888571fea 100644 --- a/cmake/macros/SundialsAddTest.cmake +++ b/cmake/macros/SundialsAddTest.cmake @@ -11,12 +11,46 @@ # SPDX-License-Identifier: BSD-3-Clause # SUNDIALS Copyright End # ------------------------------------------------------------------------------ -# -# SUNDIALS_ADD_TEST( ) + +# ~~~ +# sundials_add_test( +# [NODIFF] +# [MPI_NPROCS num_processes] +# [FLOAT_PRECISION num_digits] +# [INTEGER_PRECISION percent_difference] +# [ANSWER_DIR path] +# [ANSWER_FIEL file] +# [EXAMPLE_TYPE type] +# [TEST_ARGS arg1 arg2 ...] +# [LABELS label1 label2 ...]) +# ~~~ # # CMake macro to add a SUNDIALS regression test. Keyword input arguments can be -# added after to set regression test options (see oneValueArgs and -# multiValueArgs below). +# added after to set regression test options. +# +# The option NODIFF disables comparison of the test output against the answer +# file +# +# The option MPI_NPROCS sets the number of mpi tasks to use in parallel tests +# +# The option FLOAT_PRECISION set the precision (number of digits) for floating +# point failure comparisons. To use the default value, either don't provide the +# keyword, or provide the value "default". +# +# The option INTEGER_PRECISION sets the integer percentage difference for +# failure comparison. +# +# The option ANSWER_DIR sets the path to the directory containing the test +# answer file +# +# The option ANSWER_FILE set the name of test answer file +# +# The option EXAMPLE_TYPE set the example type i.e., release or develop examples +# +# The option TEST_ARGS are command line arguments to pass to the executable +# +# The options LABELS are labels added to the test properties to easily run (or +# exclude) groups of test with ctest -L