diff --git a/config.yaml b/config.yaml new file mode 100644 index 000000000..75b033c1f --- /dev/null +++ b/config.yaml @@ -0,0 +1,246 @@ +# ------------------------------------------------------------------------- +# This is the default spack configuration file. +# +# Settings here are versioned with Spack and are intended to provide +# sensible defaults out of the box. Spack maintainers should edit this +# file to keep it current. +# +# Users can override these settings by editing the following files. +# +# Per-spack-instance settings (overrides defaults): +# $SPACK_ROOT/etc/spack/config.yaml +# +# Per-user settings (overrides default and site settings): +# ~/.spack/config.yaml +# ------------------------------------------------------------------------- +config: + # This is the path to the root of the Spack install tree. + # You can use $spack here to refer to the root of the spack instance. + install_tree: + root: /usr/workspace/asde/qsExperimentPackages + #root: $spack/opt/spack + projections: + all: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}" + # install_tree can include an optional padded length (int or boolean) + # default is False (do not pad) + # if padded_length is True, Spack will pad as close to the system max path + # length as possible + # if padded_length is an integer, Spack will pad to that many characters, + # assuming it is higher than the length of the install_tree root. + # padded_length: 128 + + + # Locations where templates should be found + template_dirs: + - $spack/share/spack/templates + + # Directory where licenses should be located + license_dir: $spack/etc/spack/licenses + + # Temporary locations Spack can try to use for builds. + # + # Recommended options are given below. + # + # Builds can be faster in temporary directories on some (e.g., HPC) systems. + # Specifying `$tempdir` will ensure use of the default temporary directory + # (i.e., ``$TMP` or ``$TMPDIR``). + # + # Another option that prevents conflicts and potential permission issues is + # to specify `$user_cache_path/stage`, which ensures each user builds in their + # home directory. + # + # A more traditional path uses the value of `$spack/var/spack/stage`, which + # builds directly inside Spack's instance without staging them in a + # temporary space. Problems with specifying a path inside a Spack instance + # are that it precludes its use as a system package and its ability to be + # pip installable. + # + # In Spack environment files, chaining onto existing system Spack + # installations, the $env variable can be used to download, cache and build + # into user-writable paths that are relative to the currently active + # environment. + # + # In any case, if the username is not already in the path, Spack will append + # the value of `$user` in an attempt to avoid potential conflicts between + # users in shared temporary spaces. + # + # The build stage can be purged with `spack clean --stage` and + # `spack clean -a`, so it is important that the specified directory uniquely + # identifies Spack staging to avoid accidentally wiping out non-Spack work. + build_stage: + - $tempdir/$user/spack-stage + - $user_cache_path/stage + # - $spack/var/spack/stage + + # Directory in which to run tests and store test results. + # Tests will be stored in directories named by date/time and package + # name/hash. + test_stage: $user_cache_path/test + + # Cache directory for already downloaded source tarballs and archived + # repositories. This can be purged with `spack clean --downloads`. + source_cache: $spack/var/spack/cache + + + ## Directory where spack managed environments are created and stored + # environments_root: $spack/var/spack/environments + + + # Cache directory for miscellaneous files, like the package index. + # This can be purged with `spack clean --misc-cache` + misc_cache: $user_cache_path/cache + + + # Timeout in seconds used for downloading sources etc. This only applies + # to the connection phase and can be increased for slow connections or + # servers. 0 means no timeout. + connect_timeout: 10 + + + # If this is false, tools like curl that use SSL will not verify + # certifiates. (e.g., curl will use use the -k option) + verify_ssl: true + + + # This is where custom certs for proxy/firewall are stored. + # It can be a path or environment variable. To match ssl env configuration + # the default is the environment variable SSL_CERT_FILE + ssl_certs: $SSL_CERT_FILE + + + # Suppress gpg warnings from binary package verification + # Only suppresses warnings, gpg failure will still fail the install + # Potential rationale to set True: users have already explicitly trusted the + # gpg key they are using, and may not want to see repeated warnings that it + # is self-signed or something of the sort. + suppress_gpg_warnings: false + + + # If set to true, Spack will attempt to build any compiler on the spec + # that is not already available. If set to False, Spack will only use + # compilers already configured in compilers.yaml + install_missing_compilers: false + + + # If set to true, Spack will always check checksums after downloading + # archives. If false, Spack skips the checksum step. + checksum: true + + + # If set to true, Spack will fetch deprecated versions without warning. + # If false, Spack will raise an error when trying to install a deprecated version. + deprecated: false + + + # If set to true, `spack install` and friends will NOT clean + # potentially harmful variables from the build environment. Use wisely. + dirty: false + + + # The language the build environment will use. This will produce English + # compiler messages by default, so the log parser can highlight errors. + # If set to C, it will use English (see man locale). + # If set to the empty string (''), it will use the language from the + # user's environment. + build_language: C + + + # When set to true, concurrent instances of Spack will use locks to + # avoid modifying the install tree, database file, etc. If false, Spack + # will disable all locking, but you must NOT run concurrent instances + # of Spack. For filesystems that don't support locking, you should set + # this to false and run one Spack at a time, but otherwise we recommend + # enabling locks. + locks: true + + # The default url fetch method to use. + # If set to 'curl', Spack will require curl on the user's system + # If set to 'urllib', Spack will use python built-in libs to fetch + url_fetch_method: urllib + + # The maximum number of jobs to use for the build system (e.g. `make`), when + # the -j flag is not given on the command line. Defaults to 16 when not set. + # Note that the maximum number of jobs is limited by the number of cores + # available, taking thread affinity into account when supported. For instance: + # - With `build_jobs: 16` and 4 cores available `spack install` will run `make -j4` + # - With `build_jobs: 16` and 32 cores available `spack install` will run `make -j16` + # - With `build_jobs: 2` and 4 cores available `spack install -j6` will run `make -j6` + # build_jobs: 16 + + + # If set to true, Spack will use ccache to cache C compiles. + ccache: false + + + # The concretization algorithm to use in Spack. Options are: + # + # 'clingo': Uses a logic solver under the hood to solve DAGs with full + # backtracking and optimization for user preferences. Spack will + # try to bootstrap the logic solver, if not already available. + # + # 'original': Spack's original greedy, fixed-point concretizer. This + # algorithm can make decisions too early and will not backtrack + # sufficiently for many specs. This will soon be deprecated in + # favor of clingo. + # + # See `concretizer.yaml` for more settings you can fine-tune when + # using clingo. + concretizer: clingo + + + # How long to wait to lock the Spack installation database. This lock is used + # when Spack needs to manage its own package metadata and all operations are + # expected to complete within the default time limit. The timeout should + # therefore generally be left untouched. + db_lock_timeout: 60 + + + # How long to wait when attempting to modify a package (e.g. to install it). + # This value should typically be 'null' (never time out) unless the Spack + # instance only ever has a single user at a time, and only if the user + # anticipates that a significant delay indicates that the lock attempt will + # never succeed. + package_lock_timeout: null + + + # Control how shared libraries are located at runtime on Linux. See the + # the Spack documentation for details. + shared_linking: + # Spack automatically embeds runtime search paths in ELF binaries for their + # dependencies. Their type can either be "rpath" or "runpath". For glibc, rpath is + # inherited and has precedence over LD_LIBRARY_PATH; runpath is not inherited + # and of lower precedence. DO NOT MIX these within the same install tree. + type: rpath + + + # (Experimental) Embed absolute paths of dependent libraries directly in ELF + # binaries to avoid runtime search. This can improve startup time of + # executables with many dependencies, in particular on slow filesystems. + bind: false + + + # Set to 'false' to allow installation on filesystems that doesn't allow setgid bit + # manipulation by unprivileged user (e.g. AFS) + allow_sgid: true + + # Whether to show status information during building and installing packages. + # This gives information about Spack's current progress as well as the current + # and total number of packages. Information is shown both in the terminal + # title and inline. + install_status: true + + # Number of seconds a buildcache's index.json is cached locally before probing + # for updates, within a single Spack invocation. Defaults to 10 minutes. + binary_index_ttl: 600 + + flags: + # Whether to keep -Werror flags active in package builds. + keep_werror: 'none' + + # A mapping of aliases that can be used to define new commands. For instance, + # `sp: spec -I` will define a new command `sp` that will execute `spec` with + # the `-I` argument. Aliases cannot override existing commands. + aliases: + concretise: concretize + containerise: containerize + rm: remove diff --git a/createLargeDane.py b/createLargeDane.py index 407a45666..b1938a49c 100644 --- a/createLargeDane.py +++ b/createLargeDane.py @@ -11,7 +11,8 @@ f = open("daneRun.sh", "w") f.write("#!/bin/bash\n#SBATCH -n 32\n#SBATCH -N 1\n#SBATCH --time 599") -for i in range(1,2): +#add mpibind stuff here +for i in range(1,5): for progOpt in progOpts: for n_rank in n_ranks: for scale in scaling: diff --git a/setupDane.sh b/setupDane.sh index de512c50d..f1310cc0f 100644 --- a/setupDane.sh +++ b/setupDane.sh @@ -8,9 +8,11 @@ benchpark system init --dest=daneTCE rubyExp cluster=dane compiler=gcc benchpark system init --dest=daneSpack rubyExp cluster=dane compiler=gccSpack benchpark experiment init --dest=quicksilvergcc12O2weak1 quicksilver experiment=weak caliper=mpi benchpark setup quicksilvergcc12O2weak1 daneTCE workspace +cp config.yaml workspace/spack/etc/spack/defaults/config.yaml . workspace/setup.sh spack install gcc@12.1.0 -for runNum in {1..2} +spack load gcc@12.1.0 +for runNum in {1..5} do for j in ${optParams[@]} do @@ -25,6 +27,7 @@ do ramble -P -D ./workspace/quicksilver$i$j$scale$runNum/Rubyexp-957b932/workspace workspace setup else + benchpark setup quicksilver$i$j$scale$runNum daneSpack workspace ramble -P -D ./workspace/quicksilver$i$j$scale$runNum/Rubyexp-fd15164/workspace workspace setup fi diff --git a/var/sys_repo/systems/rubyExp/.system.py.swp b/var/sys_repo/systems/rubyExp/.system.py.swp deleted file mode 100644 index 9426704bd..000000000 Binary files a/var/sys_repo/systems/rubyExp/.system.py.swp and /dev/null differ diff --git a/var/sys_repo/systems/rubyExp/compilers/gccSpack/00-gcc-spack-compilers.yaml b/var/sys_repo/systems/rubyExp/compilers/gccSpack/00-gcc-spack-compilers.yaml index 086d5491d..16a1a70c5 100644 --- a/var/sys_repo/systems/rubyExp/compilers/gccSpack/00-gcc-spack-compilers.yaml +++ b/var/sys_repo/systems/rubyExp/compilers/gccSpack/00-gcc-spack-compilers.yaml @@ -2,10 +2,10 @@ compilers: - compiler: spec: gcc@12.1.0 paths: - cc: /usr/workspace/knox10/addingEx/benchpark/workspace/spack/opt/spack/linux-rhel8-cascadelake/gcc-10.3.1/gcc-12.1.0-66hd6a2pz3qd5ri64ejfhuesvqvbn75n/bin/gcc - cxx: /usr/workspace/knox10/addingEx/benchpark/workspace/spack/opt/spack/linux-rhel8-cascadelake/gcc-10.3.1/gcc-12.1.0-66hd6a2pz3qd5ri64ejfhuesvqvbn75n/bin/g++ - f77: /usr/workspace/knox10/addingEx/benchpark/workspace/spack/opt/spack/linux-rhel8-cascadelake/gcc-10.3.1/gcc-12.1.0-66hd6a2pz3qd5ri64ejfhuesvqvbn75n/bin/gfortran - fc: /usr/workspace/knox10/addingEx/benchpark/workspace/spack/opt/spack/linux-rhel8-cascadelake/gcc-10.3.1/gcc-12.1.0-66hd6a2pz3qd5ri64ejfhuesvqvbn75n/bin/gfortran + cc: /usr/workspace/asde/qsExperimentPackages/linux-rhel8-cascadelake/gcc-12.2.1/gcc-12.1.0-dgytc2lghfuh6hkdkisswas2oio3ospo/bin/gcc + cxx: /usr/workspace/asde/qsExperimentPackages/linux-rhel8-cascadelake/gcc-12.2.1/gcc-12.1.0-dgytc2lghfuh6hkdkisswas2oio3ospo/bin/g++ + f77: /usr/workspace/asde/qsExperimentPackages/linux-rhel8-cascadelake/gcc-12.2.1/gcc-12.1.0-dgytc2lghfuh6hkdkisswas2oio3ospo/bin/gfortran + fc: /usr/workspace/asde/qsExperimentPackages/linux-rhel8-cascadelake/gcc-12.2.1/gcc-12.1.0-dgytc2lghfuh6hkdkisswas2oio3ospo/bin/gfortran flags: {} operating_system: rhel8 target: x86_64 diff --git a/var/sys_repo/systems/rubyExp/system.py b/var/sys_repo/systems/rubyExp/system.py index 05098e86b..c4d398113 100644 --- a/var/sys_repo/systems/rubyExp/system.py +++ b/var/sys_repo/systems/rubyExp/system.py @@ -80,8 +80,10 @@ def compiler_configs(self): selections.append(compilers / "intel" / "00-intel-2021-6-0-compilers.yaml") elif compiler == "gccSpack": selections.append(compilers / "gccSpack" / "00-gcc-spack-compilers.yaml") + return selections + def sw_description(self): """This is somewhat vestigial, and maybe deleted later. The experiments will fail if these variables are not defined though, so for now @@ -91,11 +93,11 @@ def sw_description(self): software: packages: default-compiler: - pkg_spec: gcc + pkg_spec: gcc@12 default-mpi: pkg_spec: mvapich2 compiler-gcc: - pkg_spec: gcc + pkg_spec: gcc@12 compiler-intel: pkg_spec: intel blas: