From 52e58198bb84b43ac492d39e6e8d5d13f8f349b8 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Fri, 18 Oct 2024 18:21:29 -0500 Subject: [PATCH 01/26] mli_api additions --- doc/api/mli_api.rst | 572 ++++++++++++++++++ doc/index.rst | 1 + .../storage/dragon_feature_store.py | 4 +- 3 files changed, 575 insertions(+), 2 deletions(-) create mode 100644 doc/api/mli_api.rst diff --git a/doc/api/mli_api.rst b/doc/api/mli_api.rst new file mode 100644 index 000000000..7d02bcfb0 --- /dev/null +++ b/doc/api/mli_api.rst @@ -0,0 +1,572 @@ +************************************ +Machine Learning Infrastructure API +************************************ + + +DragonCommChannel +================= + +.. currentmodule:: smartsim._core.mli.comm.channel.dragon_channel + + +.. autosummary:: + + DragonCommChannel.channel + DragonCommChannel.send + DragonCommChannel.recv + DragonCommChannel.from_descriptor + DragonCommChannel.from_local + + +.. autoclass:: DragonCommChannel + :show-inheritance: + :members: + + +DragonFLIChannel +================ + +.. currentmodule:: smartsim._core.mli.comm.channel.dragon_fli + + +.. autosummary:: + + DragonFLIChannel.__init__ + DragonFLIChannel.send + DragonFLIChannel.send_multiple + DragonFLIChannel.recv + DragonFLIChannel.from_descriptor + +.. autoclass:: DragonFLIChannel + :show-inheritance: + :members: + + +EventBroadcaster +================ + +.. currentmodule:: smartsim._core.mli.infrastructure.comm.broadcaster + + +.. autosummary:: + + EventBroadcaster.__init__ + EventBroadcaster.name + EventBroadcaster.num_buffered + EventBroadcaster.send + +.. autoclass:: EventBroadcaster + :show-inheritance: + :members: + + +EventConsumer +============= + +.. currentmodule:: smartsim._core.mli.infrastructure.comm.consumer + +.. autosummary:: + + EventConsumer.__init__ + EventConsumer.descriptor + EventConsumer.name + EventConsumer.recv + EventConsumer.register + EventConsumer.unregister + EventConsumer.listen_once + EventConsumer.listen + + +.. autoclass:: EventConsumer + :show-inheritance: + :members: + + +.. currentmodule:: smartsim._core.mli.infrastructure.comm.event + +EventBase +========== + +.. autosummary:: + EventBase.__init__ + +.. autoclass:: EventBase + :show-inheritance: + :members: + +OnShutdownRequested +=================== + +.. autosummary:: + + OnShutdownRequested.__init__ + +.. autoclass:: OnShutdownRequested + :show-inheritance: + :members: + + +OnCreateConsumer +================ + +.. autosummary:: + + OnCreateConsumer.__init__ + +.. autoclass:: OnCreateConsumer + :show-inheritance: + :members: + + +OnRemoveConsumer +================ + +.. autosummary:: + + OnRemoveConsumer.__init__ + +.. autoclass:: OnRemoveConsumer + :show-inheritance: + :members: + + +OnWriteFeatureStore +=================== + +.. autosummary:: + + OnWriteFeatureStore.__init__ + +.. autoclass:: OnWriteFeatureStore + :show-inheritance: + :members: + + +EventProducer +============== + +.. currentmodule:: smartsim._core.mli.infrastructure.comm.producer + +.. autosummary:: + + EventProducer.send + +.. autoclass:: EventProducer + :show-inheritance: + :members: + + +.. currentmodule:: smartsim._core.mli.infrastructure.control.device_manager + +WorkerDevice +============= + +.. autosummary:: + + WorkerDevice.name + WorkerDevice.add_model + WorkerDevice.remove_model + WorkerDevice.get_model + WorkerDevice.get + +.. autoclass:: WorkerDevice + :show-inheritance: + :members: + + +DeviceManager +============== + +.. autosummary:: + + DeviceManager.get_device + +.. autoclass:: DeviceManager + :show-inheritance: + :members: + + +ConsumerRegistrationListener +============================ + +.. currentmodule:: smartsim._core.mli.infrastructure.control.listener + + +.. autosummary:: + + ConsumerRegistrationListener.__init__ + +.. autoclass:: ConsumerRegistrationListener + :show-inheritance: + :members: + + +.. currentmodule:: smartsim._core.mli.infrastructure.control.request_dispatcher + +BatchQueue +========== + +.. autosummary:: + + BatchQueue.__init__ + BatchQueue.uid + BatchQueue.model_id + BatchQueue.put + BatchQueue.ready + BatchQueue.make_disposable + BatchQueue.can_be_removed + BatchQueue.flush + BatchQueue.full + BatchQueue.empty + +.. autoclass:: BatchQueue + :show-inheritance: + :members: + + +RequestDispatcher +================= + +.. autosummary:: + + RequestDispatcher.__init__ + RequestDispatcher.has_featurestore_factory + RequestDispatcher.remove_queues + RequestDispatcher.task_queue + RequestDispatcher.dispatch + RequestDispatcher.flush_requests + +.. autoclass:: RequestDispatcher + :show-inheritance: + :members: + + + +WorkerManager +============= + +.. currentmodule:: smartsim._core.mli.infrastructure.control.worker_manager + +.. autosummary:: + + WorkerManager.__init__ + WorkerManager.has_featurestore_factory + +.. autoclass:: WorkerManager + :show-inheritance: + :members: + + + +BackboneFeatureStore +==================== + +.. currentmodule:: smartsim._core.mli.infrastructure.storage.backbone_feature_store + +.. autosummary:: + + BackboneFeatureStore.__init__ + BackboneFeatureStore.wait_timeout + BackboneFeatureStore.notification_channels + BackboneFeatureStore.backend_channel + BackboneFeatureStore.worker_queue + BackboneFeatureStore.creation_date + BackboneFeatureStore.from_writable_descriptor + BackboneFeatureStore.wait_for + BackboneFeatureStore.get_env + +.. autoclass:: BackboneFeatureStore + :show-inheritance: + :members: + + +DragonFeatureStore +================== + +.. currentmodule:: smartsim._core.mli.infrastructure.storage.dragon_feature_store + +.. autosummary:: + + DragonFeatureStore.__init__ + DragonFeatureStore.pop + DragonFeatureStore.from_descriptor + +.. autoclass:: DragonFeatureStore + :show-inheritance: + :members: + + +.. currentmodule:: smartsim._core.mli.infrastructure.storage.feature_store + +ReservedKeys +============ + +.. autosummary:: + + ReservedKeys.contains + +.. autoclass:: ReservedKeys + :show-inheritance: + :members: + + +TensorKey +========= + +.. autoclass:: TensorKey + :show-inheritance: + :members: + + +ModelKey +======== + +.. autoclass:: ModelKey + :show-inheritance: + :members: + + +FeatureStore +============ + +.. autosummary:: + + FeatureStore.__init__ + FeatureStore.descriptor + +.. autoclass:: FeatureStore + :show-inheritance: + :members: + + +TorchWorker +=========== + +.. currentmodule:: smartsim._core.mli.infrastructure.worker.torch_worker + +.. autosummary:: + + TorchWorker.load_model + TorchWorker.transform_input + TorchWorker.execute + TorchWorker.transform_output + +.. autoclass:: TorchWorker + :show-inheritance: + :members: + + +.. currentmodule:: smartsim._core.mli.infrastructure.worker.worker + +InferenceRequest +================ + +.. autosummary:: + + InferenceRequest.has_raw_model + InferenceRequest.has_model_key + InferenceRequest.has_raw_inputs + InferenceRequest.has_input_keys + InferenceRequest.has_output_keys + InferenceRequest.has_input_meta + +.. autoclass:: InferenceRequest + :show-inheritance: + :members: + + +InferenceReply +=============== + +.. autosummary:: + + InferenceReply.__init__ + InferenceReply.has_outputs + InferenceReply.has_output_keys + +.. autoclass:: InferenceReply + :show-inheritance: + :members: + + +LoadModelResult +=============== + +.. autosummary:: + + LoadModelResult.__init__ + +.. autoclass:: LoadModelResult + :show-inheritance: + :members: + + +TransformInputResult +==================== + +.. autosummary:: + + TransformInputResult.__init__ + +.. autoclass:: TransformInputResult + :show-inheritance: + :members: + + +ExecuteResult +============= + +.. autosummary:: + + ExecuteResult.__init__ + +.. autoclass:: ExecuteResult + :show-inheritance: + :members: + + +FetchInputResult +================ + +.. autosummary:: + + FetchInputResult.__init__ + +.. autoclass:: FetchInputResult + :show-inheritance: + :members: + + +TransformOutputResult +====================== + +.. autosummary:: + + TransformOutputResult.__init__ + +.. autoclass:: TransformOutputResult + :show-inheritance: + :members: + + +CreateInputBatchResult +====================== + +.. autosummary:: + + CreateInputBatchResult.__init__ + +.. autoclass:: CreateInputBatchResult + :show-inheritance: + :members: + + +FetchModelResult +================ + +.. autosummary:: + + FetchModelResult.__init__ + +.. autoclass:: FetchModelResult + :show-inheritance: + :members: + + +RequestBatch +============ + +.. autosummary:: + + RequestBatch.__init__ + RequestBatch.has_valid_requests + RequestBatch.has_raw_model + RequestBatch.raw_model + RequestBatch.input_keys + RequestBatch.output_keys + + +.. autoclass:: RequestBatch + :show-inheritance: + :members: + + +MachineLearningWorkerCore +========================= + +.. autosummary:: + + MachineLearningWorkerCore.__init__ + MachineLearningWorkerCore.deserialize_message + MachineLearningWorkerCore.prepare_outputs + MachineLearningWorkerCore.fetch_model + MachineLearningWorkerCore.fetch_inputs + MachineLearningWorkerCore.place_output + +.. autoclass:: MachineLearningWorkerCore + :show-inheritance: + :members: + + +MachineLearningWorkerBase +========================= + +.. autosummary:: + + MachineLearningWorkerBase.__init__ + MachineLearningWorkerBase.load_model + MachineLearningWorkerBase.transform_input + MachineLearningWorkerBase.execute + MachineLearningWorkerBase.transform_output + + +.. autoclass:: MachineLearningWorkerBase + :show-inheritance: + :members: + + + +.. currentmodule:: smartsim._core.mli.infrastructure.environment_loader + +EnvironmentConfigLoader +======================= + +.. autosummary:: + + EnvironmentConfigLoader.__init__ + EnvironmentConfigLoader.get_backbone + EnvironmentConfigLoader.get_queue + + +.. autoclass:: EnvironmentConfigLoader + :show-inheritance: + :members: + + +MessageHandler +============== + +.. currentmodule:: smartsim._core.mli.message_handler + +.. autosummary:: + + MessageHandler.build_tensor_descriptor + MessageHandler.build_output_tensor_descriptor + MessageHandler.build_tensor_key + MessageHandler.build_model + MessageHandler.build_model_key + MessageHandler.build_torch_request_attributes + MessageHandler.build_tf_request_attributes + MessageHandler.build_torch_response_attributes + MessageHandler.build_tf_response_attributes + MessageHandler.build_request + MessageHandler.serialize_request + MessageHandler.deserialize_request + MessageHandler.build_response + MessageHandler.serialize_response + MessageHandler.deserialize_response + +.. autoclass:: MessageHandler + :show-inheritance: + :members: + diff --git a/doc/index.rst b/doc/index.rst index 4c64712b2..4c9634902 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -39,6 +39,7 @@ ml_features dragon api/smartsim_api + api/mli_api .. toctree:: :maxdepth: 2 diff --git a/smartsim/_core/mli/infrastructure/storage/dragon_feature_store.py b/smartsim/_core/mli/infrastructure/storage/dragon_feature_store.py index 24f2221c8..a3f969571 100644 --- a/smartsim/_core/mli/infrastructure/storage/dragon_feature_store.py +++ b/smartsim/_core/mli/infrastructure/storage/dragon_feature_store.py @@ -98,8 +98,8 @@ def pop(self, key: str) -> t.Union[str, bytes, None]: """Remove the value from the dictionary and return the value. :param key: Dictionary key to retrieve - :returns: The value held at the key if it exists, otherwise `None - `""" + :returns: The value held at the key if it exists, otherwise `None` + """ try: return self._storage.pop(key) except dragon_ddict.DDictError: From a564783ae6736c4051dbf74e3e6711fbe3fadc7d Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Mon, 21 Oct 2024 16:47:29 -0500 Subject: [PATCH 02/26] added line to readthedocsyaml --- .readthedocs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 88f270ba7..e828e2f7d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -28,6 +28,7 @@ build: - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . + - export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH pre_build: - pip install typing_extensions==4.8.0 - pip install pydantic==1.10.13 From 3174e3fd326f80fd56470d86e9d76ef8ef103f1c Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Mon, 21 Oct 2024 17:33:41 -0500 Subject: [PATCH 03/26] add to run_tests yaml --- .github/workflows/run_tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 9b988520a..b43dc0d51 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -108,15 +108,17 @@ jobs: python -m pip install git+https://github.com/CrayLabs/SmartRedis.git@develop#egg=smartredis python -m pip install .[dev,mypy] + - name: Install ML Runtimes if: matrix.subset != 'dragon' run: smart build --device cpu -v - - name: Install ML Runtimes (with dragon) if: matrix.subset == 'dragon' env: SMARTSIM_DRAGON_TOKEN: ${{ secrets.DRAGON_TOKEN }} + LD_LIBRARY_PATH: $PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH + run: | if [ -n "${SMARTSIM_DRAGON_TOKEN}" ]; then smart build --device cpu -v --dragon-repo dragonhpc/dragon-nightly --dragon-version 0.10 From 89ac87e7524fc66cf90250e81f8fb65aaa8e94d4 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Mon, 21 Oct 2024 17:48:55 -0500 Subject: [PATCH 04/26] add to run_tests yaml --- .github/workflows/run_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index b43dc0d51..e9dd0f760 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -117,7 +117,6 @@ jobs: if: matrix.subset == 'dragon' env: SMARTSIM_DRAGON_TOKEN: ${{ secrets.DRAGON_TOKEN }} - LD_LIBRARY_PATH: $PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH run: | if [ -n "${SMARTSIM_DRAGON_TOKEN}" ]; then @@ -128,6 +127,7 @@ jobs: SP=$(python -c 'import site; print(site.getsitepackages()[0])')/smartsim/_core/config/dragon/.env LLP=$(cat $SP | grep LD_LIBRARY_PATH | awk '{split($0, array, "="); print array[2]}') echo "LD_LIBRARY_PATH=$LLP:$LD_LIBRARY_PATH" >> $GITHUB_ENV + export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH - name: Run mypy run: | From 8e8d46fa236d62d6062ff147d44715e978ebfed4 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Mon, 21 Oct 2024 18:09:15 -0500 Subject: [PATCH 05/26] add to run_tests yaml --- .github/workflows/run_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index e9dd0f760..b8bbbd3ee 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -107,6 +107,7 @@ jobs: run: | python -m pip install git+https://github.com/CrayLabs/SmartRedis.git@develop#egg=smartredis python -m pip install .[dev,mypy] + export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH - name: Install ML Runtimes @@ -127,7 +128,6 @@ jobs: SP=$(python -c 'import site; print(site.getsitepackages()[0])')/smartsim/_core/config/dragon/.env LLP=$(cat $SP | grep LD_LIBRARY_PATH | awk '{split($0, array, "="); print array[2]}') echo "LD_LIBRARY_PATH=$LLP:$LD_LIBRARY_PATH" >> $GITHUB_ENV - export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH - name: Run mypy run: | From b383142738bb4d4007226980744c356f9be8f3c4 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Mon, 21 Oct 2024 18:15:34 -0500 Subject: [PATCH 06/26] add to run_tests yaml --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e828e2f7d..3ebc1a19f 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -23,7 +23,7 @@ build: - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard post_create_environment: - - python -m pip install .[dev,docs] + - python -m pip install .[dev,docs,ml] - cd smartredis; python -m pip install . - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran - ln -s smartredis/examples ./examples From 7e37576bf157c9bec65fd72cf30fad8232e4a5df Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Mon, 21 Oct 2024 18:19:31 -0500 Subject: [PATCH 07/26] remove changes from run_tests --- .github/workflows/run_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index b8bbbd3ee..0824a6509 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -107,7 +107,6 @@ jobs: run: | python -m pip install git+https://github.com/CrayLabs/SmartRedis.git@develop#egg=smartredis python -m pip install .[dev,mypy] - export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH - name: Install ML Runtimes From cdc96d47c8aceb5f718dcf6893d13b5fa8027dfb Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Mon, 21 Oct 2024 18:29:39 -0500 Subject: [PATCH 08/26] add dragon build to readthedocs --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3ebc1a19f..8b3dafda0 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -28,7 +28,7 @@ build: - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - - export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH + - smart build --device cpu -v --dragon pre_build: - pip install typing_extensions==4.8.0 - pip install pydantic==1.10.13 From ab4cf3c83eb71c227d327c30ca2ffc376f8b259c Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Mon, 21 Oct 2024 19:03:12 -0500 Subject: [PATCH 09/26] update dragon version --- .readthedocs.yaml | 2 +- smartsim/_core/_cli/scripts/dragon_install.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8b3dafda0..de43f4ae8 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -23,7 +23,7 @@ build: - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard post_create_environment: - - python -m pip install .[dev,docs,ml] + - python -m pip install .[dev,docs] - cd smartredis; python -m pip install . - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran - ln -s smartredis/examples ./examples diff --git a/smartsim/_core/_cli/scripts/dragon_install.py b/smartsim/_core/_cli/scripts/dragon_install.py index 3a9358390..7a7d75f1d 100644 --- a/smartsim/_core/_cli/scripts/dragon_install.py +++ b/smartsim/_core/_cli/scripts/dragon_install.py @@ -22,7 +22,7 @@ logger = get_logger(__name__) DEFAULT_DRAGON_REPO = "DragonHPC/dragon" -DEFAULT_DRAGON_VERSION = "0.9" +DEFAULT_DRAGON_VERSION = "0.10" DEFAULT_DRAGON_VERSION_TAG = f"v{DEFAULT_DRAGON_VERSION}" _GH_TOKEN = "SMARTSIM_DRAGON_TOKEN" From 253412902bbd09a670f0c614c88da7d325c0f49e Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Tue, 22 Oct 2024 16:22:42 -0500 Subject: [PATCH 10/26] dragon env vars in readthedocs --- .readthedocs.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index de43f4ae8..fff13af08 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -28,7 +28,8 @@ build: - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - - smart build --device cpu -v --dragon + - smart build --device cpu --dragon -v + - export $(cat ./_core/config/dragon/.env | grep DRAGON_INCLUDE_DIR pre_build: - pip install typing_extensions==4.8.0 - pip install pydantic==1.10.13 From 3e8c8b49c0031c04c05c8c6daf40714ca4a3596c Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Tue, 22 Oct 2024 16:23:35 -0500 Subject: [PATCH 11/26] dragon env vars in readthedocs --- .readthedocs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index fff13af08..1bbbbd58d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -30,6 +30,7 @@ build: - cd smartdashboard; python -m pip install . - smart build --device cpu --dragon -v - export $(cat ./_core/config/dragon/.env | grep DRAGON_INCLUDE_DIR + - export LD_LIBRARY_PATH=$DRAGON_INCLUDE_DIR:$LD_LIBRARY_PATH pre_build: - pip install typing_extensions==4.8.0 - pip install pydantic==1.10.13 From 5482abb43e5a6b5df6dd523a91169b1255efafb7 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Tue, 22 Oct 2024 17:03:01 -0500 Subject: [PATCH 12/26] readthedocs update --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1bbbbd58d..504e217ef 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -29,7 +29,7 @@ build: - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - smart build --device cpu --dragon -v - - export $(cat ./_core/config/dragon/.env | grep DRAGON_INCLUDE_DIR + - export $(cat ./_core/config/dragon/.env | grep DRAGON_INCLUDE_DIR) - export LD_LIBRARY_PATH=$DRAGON_INCLUDE_DIR:$LD_LIBRARY_PATH pre_build: - pip install typing_extensions==4.8.0 From 76740c59731b3da5c50e5a8c015fec06f27ef2f8 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Tue, 22 Oct 2024 17:25:04 -0500 Subject: [PATCH 13/26] debug --- .readthedocs.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 504e217ef..e967a9091 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -29,8 +29,12 @@ build: - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - smart build --device cpu --dragon -v - - export $(cat ./_core/config/dragon/.env | grep DRAGON_INCLUDE_DIR) + - export $(cat smartsim/_core/config/dragon/.env | grep DRAGON_INCLUDE_DIR) - export LD_LIBRARY_PATH=$DRAGON_INCLUDE_DIR:$LD_LIBRARY_PATH + - echo $DRAGON_INCLUDE_DIR + - echo $LD_LIBRARY_PATH + - ls + - pwd pre_build: - pip install typing_extensions==4.8.0 - pip install pydantic==1.10.13 From 64dd4be8db3323c7931706bfbea352c54aa75741 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Tue, 22 Oct 2024 17:25:30 -0500 Subject: [PATCH 14/26] debug --- .readthedocs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e967a9091..9891d7270 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -35,6 +35,7 @@ build: - echo $LD_LIBRARY_PATH - ls - pwd + - find . -name '.env' pre_build: - pip install typing_extensions==4.8.0 - pip install pydantic==1.10.13 From 4aac535bda73b4410203e7639274eea00d626fb6 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Tue, 22 Oct 2024 17:58:24 -0500 Subject: [PATCH 15/26] debug --- .readthedocs.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 9891d7270..e58f1388e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -29,13 +29,15 @@ build: - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - smart build --device cpu --dragon -v - - export $(cat smartsim/_core/config/dragon/.env | grep DRAGON_INCLUDE_DIR) + - export $(cat /home/docs/checkouts/readthedocs.org/user_builds/smartsim/envs/755/lib/python3.10/site-packages/smartsim/_core/.dragon/.env | grep DRAGON_INCLUDE_DIR) - export LD_LIBRARY_PATH=$DRAGON_INCLUDE_DIR:$LD_LIBRARY_PATH - echo $DRAGON_INCLUDE_DIR - echo $LD_LIBRARY_PATH - ls - pwd - - find . -name '.env' + - find /home/docs/checkouts/readthedocs.org/user_builds/smartsim/envs/755/lib/python3.10/site-packages/smartsim/_core/.dragon -name ".env" + - which smart + pre_build: - pip install typing_extensions==4.8.0 - pip install pydantic==1.10.13 From dbc21fe863f9997ab102776cf6747bcfd88a7bd7 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Tue, 22 Oct 2024 17:59:55 -0500 Subject: [PATCH 16/26] debug --- .readthedocs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e58f1388e..dcbd6955e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -37,6 +37,7 @@ build: - pwd - find /home/docs/checkouts/readthedocs.org/user_builds/smartsim/envs/755/lib/python3.10/site-packages/smartsim/_core/.dragon -name ".env" - which smart + - smart site pre_build: - pip install typing_extensions==4.8.0 From da0dc23c67b4b66ce2a40aff5bfde81c150207b2 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Thu, 24 Oct 2024 16:45:02 -0500 Subject: [PATCH 17/26] readthedocstet --- .readthedocs.yaml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index dcbd6955e..086ed36b0 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,6 +19,8 @@ build: then exit 183; fi + - sudo touch /usr/lib/foo + - ls /usr/lib/foo pre_create_environment: - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard @@ -29,15 +31,18 @@ build: - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - smart build --device cpu --dragon -v - - export $(cat /home/docs/checkouts/readthedocs.org/user_builds/smartsim/envs/755/lib/python3.10/site-packages/smartsim/_core/.dragon/.env | grep DRAGON_INCLUDE_DIR) - - export LD_LIBRARY_PATH=$DRAGON_INCLUDE_DIR:$LD_LIBRARY_PATH - - echo $DRAGON_INCLUDE_DIR - - echo $LD_LIBRARY_PATH - - ls - - pwd - - find /home/docs/checkouts/readthedocs.org/user_builds/smartsim/envs/755/lib/python3.10/site-packages/smartsim/_core/.dragon -name ".env" - - which smart - - smart site + - sudo cp ./smartsim/_core/.dragon/dragon-0.9/lib/libdragon.so /usr/lib/ + + + # - export $(cat /home/docs/checkouts/readthedocs.org/user_builds/smartsim/envs/755/lib/python3.10/site-packages/smartsim/_core/.dragon/.env | grep DRAGON_INCLUDE_DIR) + # - export LD_LIBRARY_PATH=$DRAGON_INCLUDE_DIR:$LD_LIBRARY_PATH + # - echo $DRAGON_INCLUDE_DIR + # - echo $LD_LIBRARY_PATH + # - ls + # - pwd + # - find /home/docs/checkouts/readthedocs.org/user_builds/smartsim/envs/755/lib/python3.10/site-packages/smartsim/_core/.dragon -name ".env" + # - which smart + # - smart site pre_build: - pip install typing_extensions==4.8.0 From 46f1561c096816a8119f0d57b4597cb74f21b0e5 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Thu, 24 Oct 2024 16:46:07 -0500 Subject: [PATCH 18/26] readthedocstet --- .readthedocs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 086ed36b0..a264a11d8 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,7 +19,7 @@ build: then exit 183; fi - - sudo touch /usr/lib/foo + - touch /usr/lib/foo - ls /usr/lib/foo pre_create_environment: - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis @@ -31,7 +31,7 @@ build: - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - smart build --device cpu --dragon -v - - sudo cp ./smartsim/_core/.dragon/dragon-0.9/lib/libdragon.so /usr/lib/ + - cp ./smartsim/_core/.dragon/dragon-0.9/lib/libdragon.so /usr/lib/ # - export $(cat /home/docs/checkouts/readthedocs.org/user_builds/smartsim/envs/755/lib/python3.10/site-packages/smartsim/_core/.dragon/.env | grep DRAGON_INCLUDE_DIR) From 81f08fd0c558eca501cda58cf86e4616dd74649e Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Thu, 24 Oct 2024 16:47:33 -0500 Subject: [PATCH 19/26] readthedocstet --- .readthedocs.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index a264a11d8..f74a992b9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,19 +19,22 @@ build: then exit 183; fi - - touch /usr/lib/foo - - ls /usr/lib/foo + # - touch /usr/lib/foo + # - ls /usr/lib/foo pre_create_environment: - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard post_create_environment: + - pwd - python -m pip install .[dev,docs] - cd smartredis; python -m pip install . - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - - smart build --device cpu --dragon -v - - cp ./smartsim/_core/.dragon/dragon-0.9/lib/libdragon.so /usr/lib/ + # - smart build --device cpu --dragon -v + # - cp ./smartsim/_core/.dragon/dragon-0.9/lib/libdragon.so /usr/lib/ + + # - export $(cat /home/docs/checkouts/readthedocs.org/user_builds/smartsim/envs/755/lib/python3.10/site-packages/smartsim/_core/.dragon/.env | grep DRAGON_INCLUDE_DIR) From 3dac602f3e0e2fd0bde4bc3449c319891179648c Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Thu, 24 Oct 2024 16:52:39 -0500 Subject: [PATCH 20/26] readthedocstet --- .readthedocs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f74a992b9..1f981d6a1 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,8 +19,8 @@ build: then exit 183; fi - # - touch /usr/lib/foo - # - ls /usr/lib/foo + - su && touch /usr/lib/foo + - su && ls /usr/lib/foo pre_create_environment: - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard From 7cd22fdf3e7f5a5177e27a8e89988e73812c6382 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Thu, 24 Oct 2024 17:06:34 -0500 Subject: [PATCH 21/26] readthedocstet --- .readthedocs.yaml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1f981d6a1..fa0e795df 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,8 +6,9 @@ build: os: ubuntu-22.04 tools: python: "3.10" - jobs: - post_checkout: + # jobs: + commands: + # post_checkout: # Cancel building pull requests when there aren't changed in the docs directory or YAML file. # You can add any other files or directories that you'd like here as well, # like your docs requirements file, or other files that will change your docs build. @@ -19,18 +20,26 @@ build: then exit 183; fi - - su && touch /usr/lib/foo - - su && ls /usr/lib/foo - pre_create_environment: + # - su && touch /usr/lib/foo + # - su && ls /usr/lib/foo + # pre_create_environment: - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard - post_create_environment: + # post_create_environment: - pwd - python -m pip install .[dev,docs] - cd smartredis; python -m pip install . - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . + - | + export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH && + make docs && + mkdir -p _readthedocs/ && + cp -r doc/_build/html/ _readthedocs/ + + + # - smart build --device cpu --dragon -v # - cp ./smartsim/_core/.dragon/dragon-0.9/lib/libdragon.so /usr/lib/ @@ -47,11 +56,14 @@ build: # - which smart # - smart site - pre_build: + # pre_build: - pip install typing_extensions==4.8.0 - pip install pydantic==1.10.13 - python -m sphinx -b linkcheck doc/ $READTHEDOCS_OUTPUT/linkcheck + # build: + + # Build documentation in the "docs/" directory with Sphinx sphinx: configuration: doc/conf.py From b3892886c92b21d7bd1fffca89228248fdaad42a Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Thu, 24 Oct 2024 17:27:27 -0500 Subject: [PATCH 22/26] readthedocstet --- .readthedocs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index fa0e795df..be8c8f53a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -32,6 +32,7 @@ build: - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . + - smart build --device cpu --dragon -v - | export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH && make docs && From e68425ec72953c49ec7476736c99d44475473a79 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Thu, 24 Oct 2024 17:43:57 -0500 Subject: [PATCH 23/26] readthedocstet --- .readthedocs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index be8c8f53a..61c2ac6e9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -33,6 +33,7 @@ build: - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - smart build --device cpu --dragon -v + - find . -name "libdragon.so*" - | export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH && make docs && From 14f27d60fccce20a20b017fb0113f102035a12ec Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Fri, 25 Oct 2024 17:13:21 -0500 Subject: [PATCH 24/26] find libdragon test --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 61c2ac6e9..eff4716fc 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -33,7 +33,7 @@ build: - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - smart build --device cpu --dragon -v - - find . -name "libdragon.so*" + - find / -name "libdragon.so*" - | export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH && make docs && From e95e364226a158e335a0f326b8bb14930ff977ee Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Fri, 25 Oct 2024 18:04:03 -0500 Subject: [PATCH 25/26] find libdragon test --- .readthedocs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index eff4716fc..41811c63a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -27,7 +27,7 @@ build: - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard # post_create_environment: - pwd - - python -m pip install .[dev,docs] + - python -m pip install -e .[dev,docs] - cd smartredis; python -m pip install . - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran - ln -s smartredis/examples ./examples @@ -35,7 +35,7 @@ build: - smart build --device cpu --dragon -v - find / -name "libdragon.so*" - | - export LD_LIBRARY_PATH=$PWD/smartsim/_core/.dragon/dragon-0.9/lib/:$LD_LIBRARY_PATH && + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(dirname $(find $PWD -name "libdragon.so")) && make docs && mkdir -p _readthedocs/ && cp -r doc/_build/html/ _readthedocs/ From c5ccc5a5062a9e33e77c1cd0a4c11c7040e4a415 Mon Sep 17 00:00:00 2001 From: Julia Putko Date: Fri, 25 Oct 2024 19:17:13 -0500 Subject: [PATCH 26/26] find libdragon test --- .readthedocs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 41811c63a..f1e0e2b6b 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -33,7 +33,6 @@ build: - ln -s smartredis/examples ./examples - cd smartdashboard; python -m pip install . - smart build --device cpu --dragon -v - - find / -name "libdragon.so*" - | export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(dirname $(find $PWD -name "libdragon.so")) && make docs &&