Skip to content

Commit

Permalink
Feature/support dbr 15.x (#161)
Browse files Browse the repository at this point in the history 
* Adding different runtimes

* updating python version

---------

Signed-off-by: Ashok Singamaneni <[email protected]>
Co-authored-by: Ashok Singamaneni <[email protected]>
  • Loading branch information
@pariksheet @asingamaneni
pariksheet and asingamaneni authored Sep 17, 2024
1 parent 4a4beb6 commit ef2a5e0
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 10 deletions.
42 changes: 42 additions & 0 deletions brickflow/engine/compute.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,48 @@ class DuplicateClustersDefinitionError(Exception):

# Not an enum but collection of constants, generated via script
class Runtimes:
RUNTIME_15_4_X_SCALA2_12_LTS = "15.4.x-scala2.12"
RUNTIME_15_4_X_PHOTON_SCALA2_12_LTS = "15.4.x-photon-scala2.12"
RUNTIME_15_4_X_GPU_ML_SCALA2_12_LTS = "15.4.x-gpu-ml-scala2.12"
RUNTIME_15_4_X_CPU_ML_SCALA2_12_LTS = "15.4.x-cpu-ml-scala2.12"
RUNTIME_15_4_X_AARCH64_SCALA2_12_LTS = "15.4.x-aarch64-scala2.12"
RUNTIME_15_4_X_AARCH64_PHOTON_SCALA2_12_LTS = "15.4.x-aarch64-photon-scala2.12"
RUNTIME_15_3_X_SCALA2_12 = "15.3.x-scala2.12"
RUNTIME_15_3_X_PHOTON_SCALA2_12 = "15.3.x-photon-scala2.12"
RUNTIME_15_3_X_GPU_ML_SCALA2_12 = "15.3.x-gpu-ml-scala2.12"
RUNTIME_15_3_X_CPU_ML_SCALA2_12 = "15.3.x-cpu-ml-scala2.12"
RUNTIME_15_3_X_AARCH64_SCALA2_12 = "15.3.x-aarch64-scala2.12"
RUNTIME_15_3_X_AARCH64_PHOTON_SCALA2_12 = "15.3.x-aarch64-photon-scala2.12"
RUNTIME_15_2_X_SCALA2_12 = "15.2.x-scala2.12"
RUNTIME_15_2_X_PHOTON_SCALA2_12 = "15.2.x-photon-scala2.12"
RUNTIME_15_2_X_GPU_ML_SCALA2_12 = "15.2.x-gpu-ml-scala2.12"
RUNTIME_15_2_X_CPU_ML_SCALA2_12 = "15.2.x-cpu-ml-scala2.12"
RUNTIME_15_2_X_AARCH64_SCALA2_12 = "15.2.x-aarch64-scala2.12"
RUNTIME_15_2_X_AARCH64_PHOTON_SCALA2_12 = "15.2.x-aarch64-photon-scala2.12"
RUNTIME_15_1_X_SCALA2_12 = "15.1.x-scala2.12"
RUNTIME_15_1_X_PHOTON_SCALA2_12 = "15.1.x-photon-scala2.12"
RUNTIME_15_1_X_GPU_ML_SCALA2_12 = "15.1.x-gpu-ml-scala2.12"
RUNTIME_15_1_X_CPU_ML_SCALA2_12 = "15.1.x-cpu-ml-scala2.12"
RUNTIME_15_1_X_AARCH64_SCALA2_12 = "15.1.x-aarch64-scala2.12"
RUNTIME_15_1_X_AARCH64_PHOTON_SCALA2_12 = "15.1.x-aarch64-photon-scala2.12"
RUNTIME_14_3_X_SCALA2_12_LTS = "14.3.x-scala2.12"
RUNTIME_14_3_X_PHOTON_SCALA2_12_LTS = "14.3.x-photon-scala2.12"
RUNTIME_14_3_X_GPU_ML_SCALA2_12_LTS = "14.3.x-gpu-ml-scala2.12"
RUNTIME_14_3_X_CPU_ML_SCALA2_12_LTS = "14.3.x-cpu-ml-scala2.12"
RUNTIME_14_3_X_AARCH64_SCALA2_12_LTS = "14.3.x-aarch64-scala2.12"
RUNTIME_14_3_X_AARCH64_PHOTON_SCALA2_12_LTS = "14.3.x-aarch64-photon-scala2.12"
RUNTIME_14_2_X_SCALA2_12 = "14.2.x-scala2.12"
RUNTIME_14_2_X_PHOTON_SCALA2_12 = "14.2.x-photon-scala2.12"
RUNTIME_14_2_X_GPU_ML_SCALA2_12 = "14.2.x-gpu-ml-scala2.12"
RUNTIME_14_2_X_CPU_ML_SCALA2_12 = "14.2.x-cpu-ml-scala2.12"
RUNTIME_14_2_X_AARCH64_SCALA2_12 = "14.2.x-aarch64-scala2.12"
RUNTIME_14_2_X_AARCH64_PHOTON_SCALA2_12 = "14.2.x-aarch64-photon-scala2.12"
RUNTIME_14_1_X_SCALA2_12 = "14.1.x-scala2.12"
RUNTIME_14_1_X_PHOTON_SCALA2_12 = "14.1.x-photon-scala2.12"
RUNTIME_14_1_X_GPU_ML_SCALA2_12 = "14.1.x-gpu-ml-scala2.12"
RUNTIME_14_1_X_CPU_ML_SCALA2_12 = "14.1.x-cpu-ml-scala2.12"
RUNTIME_14_1_X_AARCH64_SCALA2_12 = "14.1.x-aarch64-scala2.12"
RUNTIME_14_1_X_AARCH64_PHOTON_SCALA2_12 = "14.1.x-aarch64-photon-scala2.12"
RUNTIME_11_3_X_SCALA2_12 = "11.3.x-scala2.12"
RUNTIME_11_3_X_PHOTON_SCALA2_12 = "11.3.x-photon-scala2.12"
RUNTIME_11_3_X_GPU_ML_SCALA2_12 = "11.3.x-gpu-ml-scala2.12"
Expand Down
51 changes: 42 additions & 9 deletions poetry.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ extend-exclude = '''
'''

[tool.poetry.dependencies]
python = ">=3.8,<3.11" # pyspark <3.5 does not play happy with python 3.11
python = ">=3.8,<3.12" # pyspark <3.5 does not play happy with python 3.11
Jinja2 = "3.1.2"
click = "^8.1.3"
databricks-sdk = ">=0.1.8 <1.0.0"
Expand Down Expand Up @@ -72,6 +72,7 @@ boxsdk = "^3.9.2"
cerberus-python-client = "^2.5.4"
watchdog = "<4.0.0"
requests-mock = "1.12.1"
pyspark = "^3.0.0"

[tool.poetry.group.docs.dependencies]
mdx-include = ">=1.4.1,<2.0.0"
Expand Down

0 comments on commit ef2a5e0

Please sign in to comment.