From cf245ba65f48d1e327931670b13329c216f14279 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:14:38 +0000 Subject: [PATCH 1/4] Bump scipy from 1.14.0 to 1.14.1 Bumps [scipy](https://github.com/scipy/scipy) from 1.14.0 to 1.14.1. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.14.0...v1.14.1) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3121316..27ce6b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ Package = "https://pypi.org/project/agility/" strict = [ "numpy==2.0.1", "pandas==2.2.2", - "scipy==1.14.0", + "scipy==1.14.1", "seaborn==0.13.2", ] tests = [ From f777f226240740f6ec5a82bc2f4fac94668f6bca Mon Sep 17 00:00:00 2001 From: Alexander Bonkowski Date: Tue, 27 Aug 2024 10:29:16 +0200 Subject: [PATCH 2/4] Fix notebook linting issues --- examples/00_Introduction.ipynb | 21 +- .../03_mpi/mimimise_convert_backend.ipynb | 496 +++++++++--------- .../03_mpi/mimimise_convert_backend2.ipynb | 12 +- examples/03_mpi/mpi_test.ipynb | 202 ++----- pyproject.toml | 3 + 5 files changed, 309 insertions(+), 425 deletions(-) mode change 100755 => 100644 examples/03_mpi/mpi_test.ipynb diff --git a/examples/00_Introduction.ipynb b/examples/00_Introduction.ipynb index 8b13789..cc719a3 100644 --- a/examples/00_Introduction.ipynb +++ b/examples/00_Introduction.ipynb @@ -1 +1,20 @@ - +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO:" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/03_mpi/mimimise_convert_backend.ipynb b/examples/03_mpi/mimimise_convert_backend.ipynb index 9ecee94..52c0c75 100644 --- a/examples/03_mpi/mimimise_convert_backend.ipynb +++ b/examples/03_mpi/mimimise_convert_backend.ipynb @@ -1,247 +1,249 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "02650c3f", - "metadata": {}, - "source": [ - "1. Start interacitve slurm session, e.g.: \"srun --account rwth0773 --job-name \"InteractiveJob\" --cpus-per-task 4 --mem-per-cpu 1500 --time 12:00:00 --pty zsh\"\n", - "2. Start jupyter notebook in this session.\n", - "3. Start new ssh session with douple forwarding, e.g.: \"ssh -t -t login18-2 -L 8891:localhost:8891 ssh ncm0130 -L 8891:localhost:8891\"\n", - "4. Have fun." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "7cde15cb", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2021-12-01 10:04:31.506 [IPClusterStart] Starting ipcluster with [daemonize=False]\n", - "2021-12-01 10:04:33.128 [IPClusterStart] Starting 4 engines with \n", - "2021-12-01 10:05:03.175 [IPClusterStart] Engines appear to have started successfully\n" - ] - } - ], - "source": [ - "import os\n", - "\n", - "os.system(\"ipcluster start --engines=mpi --profile=mpi -n 4 &\")" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "211502d7", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "profile: mpi\n", - "IDs: [0, 1, 2, 3]\n" - ] - } - ], - "source": [ - "import ipyparallel\n", - "\n", - "cluster = ipyparallel.Client(engine=\"mpi\", profile=\"mpi\")\n", - "cluster.block = True\n", - "print('profile:', cluster.profile)\n", - "print(\"IDs:\", cluster.ids)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "eb485bbb", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[stdout:1] Proc 1 out of 4 procs has\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "[stdout:2] Proc 2 out of 4 procs has\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "[stdout:0] Proc 0 out of 4 procs has\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "[stdout:3] Proc 3 out of 4 procs has\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%%px\n", - "from mpi4py import MPI\n", - "me = MPI.COMM_WORLD.Get_rank()\n", - "nprocs = MPI.COMM_WORLD.Get_size()\n", - "print(\"Proc %d out of %d procs has\" % (me,nprocs))" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "53f0f842", - "metadata": {}, - "outputs": [], - "source": [ - "%%px\n", - "import numpy as np\n", - "from agility.analysis import GBStructure" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "490ce9d6", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[stdout:0] LAMMPS output is captured by PyLammps wrapper\n", - "LAMMPS (29 Sep 2021 - Update 1)\n", - " using 1 OpenMP thread(s) per MPI task\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "[stdout:3] LAMMPS output is captured by PyLammps wrapper\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "[stdout:2] LAMMPS output is captured by PyLammps wrapper\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "[stdout:1] LAMMPS output is captured by PyLammps wrapper\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a8635c51b5244ca3bf0a153e4b640f0a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "%px: 0%| | 0/4 [00:00\n", + "2021-12-01 10:05:03.175 [IPClusterStart] Engines appear to have started successfully\n" + ] + } + ], + "source": [ + "from __future__ import annotations\n", + "\n", + "import os\n", + "\n", + "os.system(\"ipcluster start --engines=mpi --profile=mpi -n 4 &\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "211502d7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "profile: mpi\n", + "IDs: [0, 1, 2, 3]\n" + ] + } + ], + "source": [ + "import ipyparallel\n", + "\n", + "cluster = ipyparallel.Client(engine=\"mpi\", profile=\"mpi\")\n", + "cluster.block = True\n", + "print(\"profile:\", cluster.profile)\n", + "print(\"IDs:\", cluster.ids)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "eb485bbb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[stdout:1] Proc 1 out of 4 procs has\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "[stdout:2] Proc 2 out of 4 procs has\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "[stdout:0] Proc 0 out of 4 procs has\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "[stdout:3] Proc 3 out of 4 procs has\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%%px\n", + "from mpi4py import MPI\n", + "me = MPI.COMM_WORLD.Get_rank()\n", + "nprocs = MPI.COMM_WORLD.Get_size()\n", + "print(\"Proc %d out of %d procs has\" % (me,nprocs))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "53f0f842", + "metadata": {}, + "outputs": [], + "source": [ + "%%px\n", + "import numpy as np\n", + "from agility.analysis import GBStructure" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "490ce9d6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[stdout:0] LAMMPS output is captured by PyLammps wrapper\n", + "LAMMPS (29 Sep 2021 - Update 1)\n", + " using 1 OpenMP thread(s) per MPI task\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "[stdout:3] LAMMPS output is captured by PyLammps wrapper\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "[stdout:2] LAMMPS output is captured by PyLammps wrapper\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "[stdout:1] LAMMPS output is captured by PyLammps wrapper\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a8635c51b5244ca3bf0a153e4b640f0a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "%px: 0%| | 0/4 [00:00\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;31m# request an MPI cluster with 4 engines\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mipp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCluster\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mengines\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'mpi'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mrc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;31m# get a broadcast_view on the cluster which is best\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;31m# suited for MPI style computation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/cluster/cluster.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 873\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 874\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 875\u001b[0;31m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_context_client\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_and_connect_sync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 876\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mclient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 877\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/_async.py\u001b[0m in \u001b[0;36m_synchronize\u001b[0;34m(self, async_f, *args, **kwargs)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0mloop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mloop\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mloop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_running\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_in_thread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masync_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_asyncio_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masync_f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/_async.py\u001b[0m in \u001b[0;36m_in_thread\u001b[0;34m(self, async_f, *args, **kwargs)\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_thread_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_callback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mthread_callback\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_synchronize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0masync_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/rwthfs/rz/SW/UTIL.common/Python/3.9.1/x86_64/lib/python3.9/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 438\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mCancelledError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mFINISHED\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 440\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 441\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 442\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/rwthfs/rz/SW/UTIL.common/Python/3.9.1/x86_64/lib/python3.9/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 387\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 388\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 389\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 390\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 391\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/_async.py\u001b[0m in \u001b[0;36mthread_callback\u001b[0;34m()\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0;32masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mthread_callback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mawait\u001b[0m \u001b[0masync_f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 51\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/cluster/cluster.py\u001b[0m in \u001b[0;36mstart_and_connect\u001b[0;34m(self, n)\u001b[0m\n\u001b[1;32m 728\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect_client\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 729\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 730\u001b[0;31m await asyncio.wrap_future(\n\u001b[0m\u001b[1;32m 731\u001b[0m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for_engines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 732\u001b[0m )\n", + "\u001b[0;31mEngineError\u001b[0m: Engine set stopped: {'exit_code': 255, 'pid': 219803, 'identifier': 'ipengine-1637855041-1tp7-1637855042-218555'}" ] } ], "source": [ "import ipyparallel as ipp\n", "\n", - "def mpi_example():\n", + "\n", + "def mpi_example() -> str: # noqa: D103\n", " from mpi4py import MPI\n", " comm = MPI.COMM_WORLD\n", " return f\"Hello World from rank {comm.Get_rank()}. total ranks={comm.Get_size()}\"\n", "\n", "# request an MPI cluster with 4 engines\n", - "with ipp.Cluster(engines='mpi', n=4) as rc:\n", + "with ipp.Cluster(engines=\"mpi\", n=4) as rc:\n", " # get a broadcast_view on the cluster which is best\n", " # suited for MPI style computation\n", " view = rc.broadcast_view()\n", @@ -142,159 +156,3 @@ "nbformat": 4, "nbformat_minor": 5 } -======= -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "1f70b392", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Starting 48 engines with \n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "86e3ac8b5a16405783756d02f8bb7fde", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "sleep: 0%| | 0/25 [00:00\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "7b55f85ec1184a43aac3c2ef3800acb8", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/4 [00:00\u001B[0;34m\u001B[0m\n\u001B[1;32m 7\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 8\u001B[0m \u001B[0;31m# request an MPI cluster with 4 engines\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 9\u001B[0;31m \u001B[0;32mwith\u001B[0m \u001B[0mipp\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mCluster\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mengines\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;34m'mpi'\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mn\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m4\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0mrc\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 10\u001B[0m \u001B[0;31m# get a broadcast_view on the cluster which is best\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 11\u001B[0m \u001B[0;31m# suited for MPI style computation\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/cluster/cluster.py\u001B[0m in \u001B[0;36m__enter__\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 873\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 874\u001B[0m \u001B[0;32mdef\u001B[0m \u001B[0m__enter__\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 875\u001B[0;31m \u001B[0mclient\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_context_client\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mstart_and_connect_sync\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 876\u001B[0m \u001B[0;32mreturn\u001B[0m \u001B[0mclient\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 877\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/_async.py\u001B[0m in \u001B[0;36m_synchronize\u001B[0;34m(self, async_f, *args, **kwargs)\u001B[0m\n\u001B[1;32m 68\u001B[0m \u001B[0mloop\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;32mNone\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 69\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mloop\u001B[0m \u001B[0;32mand\u001B[0m \u001B[0mloop\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mis_running\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 70\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_in_thread\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0masync_f\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 71\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 72\u001B[0m \u001B[0;32mreturn\u001B[0m \u001B[0m_asyncio_run\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0masync_f\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/_async.py\u001B[0m in \u001B[0;36m_in_thread\u001B[0;34m(self, async_f, *args, **kwargs)\u001B[0m\n\u001B[1;32m 53\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 54\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_thread_loop\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0madd_callback\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mthread_callback\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 55\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mfuture\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mresult\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 56\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 57\u001B[0m \u001B[0;32mdef\u001B[0m \u001B[0m_synchronize\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0masync_f\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/rwthfs/rz/SW/UTIL.common/Python/3.9.1/x86_64/lib/python3.9/concurrent/futures/_base.py\u001B[0m in \u001B[0;36mresult\u001B[0;34m(self, timeout)\u001B[0m\n\u001B[1;32m 438\u001B[0m \u001B[0;32mraise\u001B[0m \u001B[0mCancelledError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 439\u001B[0m \u001B[0;32melif\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_state\u001B[0m \u001B[0;34m==\u001B[0m \u001B[0mFINISHED\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 440\u001B[0;31m \u001B[0;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m__get_result\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 441\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 442\u001B[0m \u001B[0;32mraise\u001B[0m \u001B[0mTimeoutError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/rwthfs/rz/SW/UTIL.common/Python/3.9.1/x86_64/lib/python3.9/concurrent/futures/_base.py\u001B[0m in \u001B[0;36m__get_result\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 387\u001B[0m \u001B[0;32mdef\u001B[0m \u001B[0m__get_result\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 388\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_exception\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 389\u001B[0;31m \u001B[0;32mraise\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_exception\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 390\u001B[0m \u001B[0;32melse\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 391\u001B[0m \u001B[0;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_result\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/_async.py\u001B[0m in \u001B[0;36mthread_callback\u001B[0;34m()\u001B[0m\n\u001B[1;32m 48\u001B[0m \u001B[0;32masync\u001B[0m \u001B[0;32mdef\u001B[0m \u001B[0mthread_callback\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 49\u001B[0m \u001B[0;32mtry\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 50\u001B[0;31m \u001B[0mfuture\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mset_result\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;32mawait\u001B[0m \u001B[0masync_f\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m*\u001B[0m\u001B[0margs\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m 51\u001B[0m \u001B[0;32mexcept\u001B[0m \u001B[0mException\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0me\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 52\u001B[0m \u001B[0mfuture\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mset_exception\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0me\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/cluster/cluster.py\u001B[0m in \u001B[0;36mstart_and_connect\u001B[0;34m(self, n)\u001B[0m\n\u001B[1;32m 728\u001B[0m \u001B[0mclient\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;32mawait\u001B[0m \u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mconnect_client\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 729\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mn\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m--> 730\u001B[0;31m await asyncio.wrap_future(\n\u001B[0m\u001B[1;32m 731\u001B[0m \u001B[0mclient\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mwait_for_engines\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mn\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mblock\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mFalse\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mtimeout\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mengine_timeout\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m 732\u001B[0m )\n", - "\u001B[0;31mEngineError\u001B[0m: Engine set stopped: {'exit_code': 255, 'pid': 219803, 'identifier': 'ipengine-1637855041-1tp7-1637855042-218555'}" - ] - } - ], - "source": [ - "import ipyparallel as ipp\n", - "\n", - "def mpi_example():\n", - " from mpi4py import MPI\n", - " comm = MPI.COMM_WORLD\n", - " return f\"Hello World from rank {comm.Get_rank()}. total ranks={comm.Get_size()}\"\n", - "\n", - "# request an MPI cluster with 4 engines\n", - "with ipp.Cluster(engines='mpi', n=4) as rc:\n", - " # get a broadcast_view on the cluster which is best\n", - " # suited for MPI style computation\n", - " view = rc.broadcast_view()\n", - " # run the mpi_example function on all engines in parallel\n", - " r = view.apply_sync(mpi_example)\n", - " # Retrieve and print the result from the engines\n", - " print(\"\\n\".join(r))\n", - "# at this point, the cluster processes have been shutdown" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} ->>>>>>> b6da64d4f9eac398b797451b7a31ab24749bdd8c diff --git a/pyproject.toml b/pyproject.toml index 27ce6b9..664ac0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,9 @@ lint.ignore = [ lint.pydocstyle.convention = "google" lint.isort.required-imports = ["from __future__ import annotations"] +[tool.ruff.lint.per-file-ignores] +"*.ipynb" = ["S", "T201"] + [tool.mypy] ignore_missing_imports = true From c2152d7361988b293d9fcbbe0dfc10f31d04e082 Mon Sep 17 00:00:00 2001 From: Alexander Bonkowski Date: Tue, 27 Aug 2024 10:38:24 +0200 Subject: [PATCH 3/4] Temporarily remote `mpi_test.ipynb` --- examples/03_mpi/mpi_test.ipynb | 158 --------------------------------- 1 file changed, 158 deletions(-) delete mode 100644 examples/03_mpi/mpi_test.ipynb diff --git a/examples/03_mpi/mpi_test.ipynb b/examples/03_mpi/mpi_test.ipynb deleted file mode 100644 index 0694adf..0000000 --- a/examples/03_mpi/mpi_test.ipynb +++ /dev/null @@ -1,158 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "1f70b392", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Starting 48 engines with \n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "86e3ac8b5a16405783756d02f8bb7fde", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "sleep: 0%| | 0/25 [00:00\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "7b55f85ec1184a43aac3c2ef3800acb8", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/4 [00:00\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;31m# request an MPI cluster with 4 engines\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mipp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCluster\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mengines\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'mpi'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mrc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;31m# get a broadcast_view on the cluster which is best\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;31m# suited for MPI style computation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/cluster/cluster.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 873\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 874\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 875\u001b[0;31m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_context_client\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_and_connect_sync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 876\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mclient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 877\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/_async.py\u001b[0m in \u001b[0;36m_synchronize\u001b[0;34m(self, async_f, *args, **kwargs)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0mloop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mloop\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mloop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_running\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_in_thread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masync_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0m_asyncio_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masync_f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/_async.py\u001b[0m in \u001b[0;36m_in_thread\u001b[0;34m(self, async_f, *args, **kwargs)\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_thread_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_callback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mthread_callback\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_synchronize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0masync_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/rwthfs/rz/SW/UTIL.common/Python/3.9.1/x86_64/lib/python3.9/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 438\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mCancelledError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 439\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mFINISHED\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 440\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 441\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 442\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/rwthfs/rz/SW/UTIL.common/Python/3.9.1/x86_64/lib/python3.9/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 387\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 388\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 389\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 390\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 391\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/_async.py\u001b[0m in \u001b[0;36mthread_callback\u001b[0;34m()\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0;32masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mthread_callback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mawait\u001b[0m \u001b[0masync_f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 51\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/rwthfs/rz/cluster/home/ab542498/python/base-3.9/lib/python3.9/site-packages/ipyparallel/cluster/cluster.py\u001b[0m in \u001b[0;36mstart_and_connect\u001b[0;34m(self, n)\u001b[0m\n\u001b[1;32m 728\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect_client\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 729\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 730\u001b[0;31m await asyncio.wrap_future(\n\u001b[0m\u001b[1;32m 731\u001b[0m \u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for_engines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine_timeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 732\u001b[0m )\n", - "\u001b[0;31mEngineError\u001b[0m: Engine set stopped: {'exit_code': 255, 'pid': 219803, 'identifier': 'ipengine-1637855041-1tp7-1637855042-218555'}" - ] - } - ], - "source": [ - "import ipyparallel as ipp\n", - "\n", - "\n", - "def mpi_example() -> str: # noqa: D103\n", - " from mpi4py import MPI\n", - " comm = MPI.COMM_WORLD\n", - " return f\"Hello World from rank {comm.Get_rank()}. total ranks={comm.Get_size()}\"\n", - "\n", - "# request an MPI cluster with 4 engines\n", - "with ipp.Cluster(engines=\"mpi\", n=4) as rc:\n", - " # get a broadcast_view on the cluster which is best\n", - " # suited for MPI style computation\n", - " view = rc.broadcast_view()\n", - " # run the mpi_example function on all engines in parallel\n", - " r = view.apply_sync(mpi_example)\n", - " # Retrieve and print the result from the engines\n", - " print(\"\\n\".join(r))\n", - "# at this point, the cluster processes have been shutdown" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From b61dca1005ebd21ce193ccad0d36b966dc37d4cd Mon Sep 17 00:00:00 2001 From: Alexander Bonkowski Date: Tue, 27 Aug 2024 12:05:54 +0200 Subject: [PATCH 4/4] Add `nbstripout` --- .pre-commit-config.yaml | 6 +++++ .../03_mpi/mimimise_convert_backend.ipynb | 26 +++++++++---------- .../03_mpi/mimimise_convert_backend2.ipynb | 24 ++++++++--------- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e41c48f..3739cba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,3 +46,9 @@ repos: # MD033: no inline HTML # MD041: first line in a file should be a top-level heading args: [--disable, MD013, MD033, MD041, "--"] + + - repo: https://github.com/kynan/nbstripout + rev: 0.7.1 + hooks: + - id: nbstripout + args: [--drop-empty-cells, --keep-output] diff --git a/examples/03_mpi/mimimise_convert_backend.ipynb b/examples/03_mpi/mimimise_convert_backend.ipynb index 52c0c75..c517faf 100644 --- a/examples/03_mpi/mimimise_convert_backend.ipynb +++ b/examples/03_mpi/mimimise_convert_backend.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "02650c3f", + "id": "0", "metadata": {}, "source": [ "1. Start interacitve slurm session, e.g.: \"srun --account rwth0773 --job-name \"InteractiveJob\" --cpus-per-task 4 --mem-per-cpu 1500 --time 12:00:00 --pty zsh\"\n", @@ -13,8 +13,8 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "7cde15cb", + "execution_count": null, + "id": "1", "metadata": {}, "outputs": [ { @@ -23,7 +23,7 @@ "0" ] }, - "execution_count": 1, + "execution_count": null, "metadata": {}, "output_type": "execute_result" }, @@ -47,8 +47,8 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "211502d7", + "execution_count": null, + "id": "2", "metadata": {}, "outputs": [ { @@ -71,8 +71,8 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "eb485bbb", + "execution_count": null, + "id": "3", "metadata": {}, "outputs": [ { @@ -122,8 +122,8 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "53f0f842", + "execution_count": null, + "id": "4", "metadata": {}, "outputs": [], "source": [ @@ -134,8 +134,8 @@ }, { "cell_type": "code", - "execution_count": 9, - "id": "490ce9d6", + "execution_count": null, + "id": "5", "metadata": {}, "outputs": [ { @@ -201,7 +201,7 @@ { "cell_type": "code", "execution_count": null, - "id": "233ef59b", + "id": "6", "metadata": {}, "outputs": [ { diff --git a/examples/03_mpi/mimimise_convert_backend2.ipynb b/examples/03_mpi/mimimise_convert_backend2.ipynb index 2d1c8f3..40da892 100644 --- a/examples/03_mpi/mimimise_convert_backend2.ipynb +++ b/examples/03_mpi/mimimise_convert_backend2.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "9ded445e", + "id": "0", "metadata": {}, "source": [ "1. Start interacitve slurm session, e.g.: \"srun --account rwth0773 --job-name \"InteractiveJob\" --cpus-per-task 4 --mem-per-cpu 1500 --time 12:00:00 --pty zsh\"\n", @@ -13,8 +13,8 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "7cde15cb", + "execution_count": null, + "id": "1", "metadata": {}, "outputs": [ { @@ -23,7 +23,7 @@ "0" ] }, - "execution_count": 1, + "execution_count": null, "metadata": {}, "output_type": "execute_result" } @@ -38,8 +38,8 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "211502d7", + "execution_count": null, + "id": "2", "metadata": {}, "outputs": [ { @@ -62,8 +62,8 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "eb485bbb", + "execution_count": null, + "id": "3", "metadata": {}, "outputs": [ { @@ -113,8 +113,8 @@ }, { "cell_type": "code", - "execution_count": 5, - "id": "53f0f842", + "execution_count": null, + "id": "4", "metadata": {}, "outputs": [ { @@ -140,8 +140,8 @@ }, { "cell_type": "code", - "execution_count": 6, - "id": "490ce9d6", + "execution_count": null, + "id": "5", "metadata": {}, "outputs": [ {