Skip to content

Commit

Permalink
Add PDM and PythonContainer executable
Browse files Browse the repository at this point in the history
  • Loading branch information
ethanluoyc committed Nov 5, 2023
1 parent 8a44671 commit e7ce5f9
Show file tree
Hide file tree
Showing 18 changed files with 641 additions and 0 deletions.
162 changes: 162 additions & 0 deletions examples/pdm_project/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version

# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock

# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock

# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
.pdm-python
.pdm-build/

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
1 change: 1 addition & 0 deletions examples/pdm_project/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# pdm_project
47 changes: 47 additions & 0 deletions examples/pdm_project/launcher.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
from absl import app
from absl import flags

from lxm3 import xm
from lxm3 import xm_cluster
from lxm3.contrib import ucl

_LAUNCH_ON_CLUSTER = flags.DEFINE_boolean(
"launch_on_cluster", False, "Launch on cluster"
)


def main(_):
with xm_cluster.create_experiment(experiment_title="basic") as experiment:
job_requirements = xm_cluster.JobRequirements(ram=8 * xm.GB)
if _LAUNCH_ON_CLUSTER.value:
# This is a special case for using SGE in UCL where we use generic
# job requirements and translate to SGE specific requirements.
# Non-UCL users, use `xm_cluster.GridEngine directly`.
executor = ucl.UclGridEngine(
job_requirements,
walltime=10 * xm.Min,
)
else:
executor = xm_cluster.Local(job_requirements)

spec = xm_cluster.PDMProject(
# This is a relative path to the launcher that contains
# your python package (i.e. the directory that contains pyproject.toml)
path=".",
base_image="python:3.10-slim",
# Entrypoint is the python module that you would like to
# In the implementation, this is translated to
# python3 -m py_package.main
entrypoint=xm_cluster.ModuleName("pdm_project.main"),
)

[executable] = experiment.package(
[xm.Packageable(spec, executor_spec=executor.Spec())]
)

experiment.add(xm.Job(executable=executable, executor=executor))


if __name__ == "__main__":
app.run(main)
8 changes: 8 additions & 0 deletions examples/pdm_project/pdm.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions examples/pdm_project/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
[project]
name = "pdm_project"
version = "0.1.0"
description = ""
authors = [
{name = "Yicheng Luo", email = "[email protected]"},
]
dependencies = []
requires-python = ">=3.10"
readme = "README.md"
license = {text = "MIT"}

[build-system]
requires = ["pdm-backend"]
build-backend = "pdm.backend"
Empty file.
1 change: 1 addition & 0 deletions examples/pdm_project/src/pdm_project/main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
print("Hello world")
Empty file.
Loading

0 comments on commit e7ce5f9

Please sign in to comment.