Generate the requirements files from dependencies.yaml

Signed-off-by: Anuradha Karuppiah <[email protected]>

ci/conda/recipes/morpheus-libs/morpheus_core_test.sh

@@ -13,11 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# morpheus tests are dependent on some pypi packages
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_core --matrix "" | tee "core_requirements.txt"
-
-pip install -r core_requirements.txt
+python3 <<EOF
+import pkgutil
+import subprocess
+data = pkgutil.get_data("morpheus", "requirements_morpheus_core.txt")
+requirements = data.decode("utf-8")
+subprocess.call(f"pip install {requirements}".split())
+EOF
 
 pytest tests/morpheus

ci/conda/recipes/morpheus-libs/morpheus_dfp_test.sh

@@ -13,11 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# morpheus tests are dependent on some pypi packages
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_dfp --matrix "" | tee "dfp_requirements.txt"
-
-pip install -r dfp_requirements.txt
+python3 <<EOF
+import pkgutil
+import subprocess
+data = pkgutil.get_data("morpheus_dfp", "requirements_morpheus_dfp.txt")
+requirements = data.decode("utf-8")
+subprocess.call(f"pip install {requirements}".split())
+EOF
 
 pytest tests/morpheus_dfp

ci/conda/recipes/morpheus-libs/morpheus_llm_test.sh

@@ -13,23 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# llm library and tests are dependent on a number of pypi packages - fixme
-rapids-dependency-file-generator \
-  --output requirements \
-  --file-key morpheus_llm --matrix "" | tee "llm_requirements.txt"
-
-pip install -r llm_requirements.txt
-
 # Install requirements if they are included in the package
 python3 <<EOF
 import pkgutil
 import subprocess
-try:
-    data = pkgutil.get_data("morpheus_llm", "requirements.txt")
-    requirements = data.decode("utf-8")
-    subprocess.call(f"pip install {requirements}".split())
-except FileNotFoundError:
-    pass
+data = pkgutil.get_data("morpheus_llm", "requirements_morpheus_llm.txt")
+requirements = data.decode("utf-8")
+subprocess.call(f"pip install {requirements}".split())
 EOF
 
 pytest tests/morpheus_llm

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -43,6 +43,7 @@ dependencies:
 - doxygen=1.9.2
 - elasticsearch==8.9.0
 - exhale=0.3.6
+- faiss-cpu
 - feedparser=6.0
 - flake8
 - git-lfs
@@ -132,7 +133,6 @@ dependencies:
   - databricks-connect
   - dgl==2.0.0
   - dglgo
-  - faiss-cpu
   - google-search-results==2.4
   - langchain-nvidia-ai-endpoints==0.0.11
   - langchain==0.1.16

conda/environments/examples_cuda-125_arch-x86_64.yaml

@@ -22,6 +22,7 @@ dependencies:
 - dill=0.3.7
 - docker-py=5.0
 - elasticsearch==8.9.0
+- faiss-cpu
 - feedparser=6.0
 - grpcio
 - grpcio-status
@@ -71,7 +72,6 @@ dependencies:
   - databricks-connect
   - dgl==2.0.0
   - dglgo
-  - faiss-cpu
   - google-search-results==2.4
   - langchain-nvidia-ai-endpoints==0.0.11
   - langchain==0.1.16

dependencies.yaml

@@ -176,23 +176,27 @@ files:
       - checks
       - python
 
-  # pip dependencies that are used during the conda test stage for morpheus-llm output
-  morpheus_llm:
-    output: none
-    includes:
-      - test_morpheus_core_pip
-
-  # pip dependencies that are used during the conda test stage for morpheus-core output
+  # morpheus-core pip dependencies are published as a morpheus package data file
   morpheus_core:
-    output: none
+    output: requirements
+    requirements_dir: python/morpheus/morpheus
     includes:
-      - test_morpheus_core_pip
+      - morpheus_core_pip
 
-  # pip dependencies that are used during the conda test stage for morpheus-dfp output
+  # morpheus-dfp pip dependencies are published as a morpheus_dfp package data file
   morpheus_dfp:
-    output: none
+    output: requirements
+    requirements_dir: python/morpheus_dfp/morpheus_dfp
     includes:
-      - test_morpheus_core_pip
+      - morpheus_core_pip
+
+  # morpheus-llm pip dependencies are published as a morpheus_llm package data file
+  morpheus_llm:
+    output: requirements
+    requirements_dir: python/morpheus_llm/morpheus_llm
+    includes:
+      - morpheus_core_pip
+      - morpheus_llm_pip
 
 channels:
   - conda-forge
@@ -401,14 +405,13 @@ dependencies:
           - pip:
               - pytest-kafka==0.6.0
 
-  test_morpheus_core_pip:
+  morpheus_core_pip:
     common:
       - output_types: [requirements]
         packages:
            - --extra-index-url https://download.pytorch.org/whl/cu124
            - torch==2.4.0+cu124
 
-
   morpheus_llm_pip:
     common:
       - output_types: [requirements]

python/morpheus/MANIFEST.in

@@ -1,3 +1,4 @@
 include morpheus/_version.py
+include morpheus/requirements_morpheus_core.txt
 recursive-include morpheus/data *
 recursive-include morpheus *.cpython*.so py.typed *.pyi

python/morpheus/morpheus/requirements_morpheus_core.txt

@@ -0,0 +1,4 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+torch==2.4.0+cu124

python/morpheus_dfp/MANIFEST.in

@@ -1,2 +1,3 @@
 include morpheus_dfp/_version.py
+include morpheus_dfp/requirements_morpheus_dfp.txt
 recursive-include morpheus_dfp py.typed

python/morpheus_dfp/morpheus_dfp/requirements_morpheus_dfp.txt

@@ -0,0 +1,4 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+torch==2.4.0+cu124

python/morpheus_llm/MANIFEST.in

@@ -1,3 +1,3 @@
 include morpheus_llm/_version.py
-include morpheus_llm/requirements.txt
+include morpheus_llm/requirements_morpheus_llm.txt
 recursive-include morpheus_llm *.cpython*.so py.typed *.pyi

python/morpheus_llm/morpheus_llm/requirements_morpheus_llm.txt

@@ -0,0 +1,10 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`.
+--extra-index-url https://download.pytorch.org/whl/cu124
+google-search-results==2.4
+langchain-nvidia-ai-endpoints==0.0.11
+langchain==0.1.16
+milvus==2.3.5
+nemollm==0.3.5
+pymilvus==2.3.6
+torch==2.4.0+cu124