diff --git a/.gitignore b/.gitignore index 7c5d9ca0..8bb9771a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.vscode/ tmp/ /tests/basic/.git /docs/build @@ -11,4 +12,5 @@ venv /py_track.egg-info/ __pycache__/ .pymon -.ipynb_checkpoints \ No newline at end of file +.ipynb_checkpoints +_examples/ diff --git a/examples/dataclasses_for_parameters.ipynb b/examples/dataclasses_for_parameters.ipynb index 53fd1a1c..08191328 100644 --- a/examples/dataclasses_for_parameters.ipynb +++ b/examples/dataclasses_for_parameters.ipynb @@ -17,97 +17,103 @@ { "cell_type": "code", "execution_count": 1, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "import pathlib\n", "\n", "from zntrack import config\n", + "import znjson\n", + "import dataclasses\n", "\n", "config.nb_name = \"dataclasses_for_parameters.ipynb\"" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 2, "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 2, + }, "outputs": [], "source": [ "from zntrack.utils import cwd_temp_dir\n", "\n", "temp_dir = cwd_temp_dir()" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 3, "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 3, + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Initialized empty Git repository in /tmp/tmpumpig7qo/.git/\r\n", - "Initialized DVC repository.\r\n", - "\r\n", - "You can now commit the changes to git.\r\n", - "\r\n", - "\u001B[31m+---------------------------------------------------------------------+\r\n", - "\u001B[0m\u001B[31m|\u001B[0m \u001B[31m|\u001B[0m\r\n", - "\u001B[31m|\u001B[0m DVC has enabled anonymous aggregate usage analytics. \u001B[31m|\u001B[0m\r\n", - "\u001B[31m|\u001B[0m Read the analytics documentation (and how to opt-out) here: \u001B[31m|\u001B[0m\r\n", - "\u001B[31m|\u001B[0m <\u001B[36mhttps://dvc.org/doc/user-guide/analytics\u001B[39m> \u001B[31m|\u001B[0m\r\n", - "\u001B[31m|\u001B[0m \u001B[31m|\u001B[0m\r\n", - "\u001B[31m+---------------------------------------------------------------------+\r\n", - "\u001B[0m\r\n", - "\u001B[33mWhat's next?\u001B[39m\r\n", - "\u001B[33m------------\u001B[39m\r\n", - "- Check out the documentation: <\u001B[36mhttps://dvc.org/doc\u001B[39m>\r\n", - "- Get help and share ideas: <\u001B[36mhttps://dvc.org/chat\u001B[39m>\r\n", - "- Star us on GitHub: <\u001B[36mhttps://github.com/iterative/dvc\u001B[39m>\r\n", - "\u001B[0m" + "Initialized empty Git repository in /tmp/tmpv_5nwynt/.git/\n", + "Initialized DVC repository.\n", + "\n", + "You can now commit the changes to git.\n", + "\n", + "+---------------------------------------------------------------------+\n", + "| |\n", + "| DVC has enabled anonymous aggregate usage analytics. |\n", + "| Read the analytics documentation (and how to opt-out) here: |\n", + "| |\n", + "| |\n", + "+---------------------------------------------------------------------+\n", + "\n", + "What's next?\n", + "------------\n", + "- Check out the documentation: \n", + "- Get help and share ideas: \n", + "- Star us on GitHub: \n" ] } ], "source": [ "!git init\n", "!dvc init" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] }, { "cell_type": "code", "execution_count": 4, - "outputs": [], - "source": [ - "import dataclasses\n", - "from zntrack import Node, zn\n", - "import random" - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } + }, + "outputs": [], + "source": [ + "from zntrack import Node, zn, Project\n", + "import random" + ] }, { "cell_type": "code", "execution_count": 5, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [], "source": [ "@dataclasses.dataclass\n", @@ -116,114 +122,188 @@ " stop: int\n", " step: int = 1\n", "\n", + "class ParameterConverter(znjson.ConverterBase):\n", + " instance = Parameter\n", + " representation = \"Parameter\"\n", + "\n", + " def encode(self, obj) -> str:\n", + " return dataclasses.asdict(obj)\n", + " \n", + " def decode(self, value: str):\n", + " return Parameter(**value)\n", "\n", "class ComputeRandomNumber(Node):\n", - " parameter: Parameter = zn.Method()\n", + " param: Parameter = zn.params()\n", " number = zn.outs()\n", "\n", - " def __init__(self, parameter: Parameter = None, **kwargs):\n", + " # register the Converter\n", + " # ----------------------\n", + " # you can register the converter anywhere,\n", + " # as long as it is executed before\n", + " # the first time the node is instantiated\n", + " _ = znjson.config.register(ParameterConverter)\n", + "\n", + " def __init__(self, param: Parameter = None, **kwargs):\n", " super().__init__(**kwargs)\n", - " self.parameter = parameter\n", + " self.param = param\n", "\n", " def run(self):\n", " self.number = random.randrange(\n", - " self.parameter.start, self.parameter.stop, self.parameter.step\n", + " self.param.start, self.param.stop, self.param.step\n", " )" - ], + ] + }, + { + "cell_type": "code", + "execution_count": 6, "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 6, + }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Running DVC command: 'stage add --name ComputeRandomNumber --force ...'\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "2022-02-22 13:54:17,781 (WARNING): Jupyter support is an experimental feature! Please save your notebook before running this command!\n", + "Creating 'dvc.yaml'\n", + "Adding stage 'ComputeRandomNumber' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add nodes/ComputeRandomNumber/.gitignore dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Jupyter support is an experimental feature! Please save your notebook before running this command!\n", "Submit issues to https://github.com/zincware/ZnTrack.\n", - "2022-02-22 13:54:21,215 (WARNING): Running DVC command: 'dvc run -n ComputeRandomNumber ...'\n" + "[NbConvertApp] Converting notebook dataclasses_for_parameters.ipynb to script\n", + "[NbConvertApp] Writing 1907 bytes to dataclasses_for_parameters.py\n", + "Running DVC command: 'repro'\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running stage 'ComputeRandomNumber':\n", + "> zntrack run src.ComputeRandomNumber.ComputeRandomNumber --name ComputeRandomNumber\n", + "Generating lock file 'dvc.lock'\n", + "Updating lock file 'dvc.lock'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.lock\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n", + "Use `dvc push` to send your updates to remote storage.\n" ] } ], "source": [ - "ComputeRandomNumber(parameter=Parameter(start=100, stop=200)).write_graph(no_exec=False)" - ], + "parameter = Parameter(start=100, stop=200)\n", + "with Project() as proj:\n", + " ComputeRandomNumber(param=parameter)\n", + "proj.run(repro=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, "metadata": { "collapsed": false, "pycharm": { "name": "#%%\n" } - } - }, - { - "cell_type": "code", - "execution_count": 7, + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "146\n", + "134\n", "Parameter(start=100, stop=200, step=1)\n" ] } ], "source": [ - "print(ComputeRandomNumber.load().number)\n", - "print(ComputeRandomNumber.load().parameter)" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + "print(ComputeRandomNumber.from_rev().number)\n", + "print(ComputeRandomNumber.from_rev().param)" + ] }, { "cell_type": "markdown", - "source": [ - "The arguments of the dataclass are saved in the `params.yaml` file and can also be modified there." - ], "metadata": { "collapsed": false, "pycharm": { "name": "#%% md\n" } - } + }, + "source": [ + "The arguments of the dataclass are saved in the `params.yaml` file and can also be modified there." + ] }, { "cell_type": "code", "execution_count": 8, + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ComputeRandomNumber:\n", - " parameter:\n", - " _cls: Parameter\n", - " start: 100\n", - " step: 1\n", - " stop: 200\n", + " param:\n", + " _type: Parameter\n", + " value:\n", + " start: 100\n", + " step: 1\n", + " stop: 200\n", "\n" ] } ], "source": [ "print(pathlib.Path(\"params.yaml\").read_text())" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } - } + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "temp_dir.cleanup()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -235,16 +315,16 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/examples/docs/01_Intro.ipynb b/examples/docs/01_Intro.ipynb index cacbe23e..7e8caf31 100644 --- a/examples/docs/01_Intro.ipynb +++ b/examples/docs/01_Intro.ipynb @@ -24,6 +24,9 @@ }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "## Jupyter Notebook Support\n", "\n", @@ -32,10 +35,7 @@ "Therefore, it needs to know the name of the notebook.\n", "\n", "For more complex workflows, it is recommended to define the Nodes inside Python files and import them into Jupyter Notebooks." - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", @@ -62,15 +62,15 @@ { "cell_type": "code", "execution_count": 2, + "metadata": { + "collapsed": false + }, "outputs": [], "source": [ "from zntrack.utils import cwd_temp_dir\n", "\n", "temp_dir = cwd_temp_dir()" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", @@ -86,24 +86,24 @@ "name": "stdout", "output_type": "stream", "text": [ - "Initialized empty Git repository in /tmp/tmpg0398qas/.git/\r\n", - "Initialized DVC repository.\r\n", - "\r\n", - "You can now commit the changes to git.\r\n", - "\r\n", - "+---------------------------------------------------------------------+\r\n", - "| |\r\n", - "| DVC has enabled anonymous aggregate usage analytics. |\r\n", - "| Read the analytics documentation (and how to opt-out) here: |\r\n", - "| |\r\n", - "| |\r\n", - "+---------------------------------------------------------------------+\r\n", - "\r\n", - "What's next?\r\n", - "------------\r\n", - "- Check out the documentation: \r\n", - "- Get help and share ideas: \r\n", - "- Star us on GitHub: \r\n" + "Initialized empty Git repository in /tmp/tmpkyrcn10i/.git/\n", + "Initialized DVC repository.\n", + "\n", + "You can now commit the changes to git.\n", + "\n", + "+---------------------------------------------------------------------+\n", + "| |\n", + "| DVC has enabled anonymous aggregate usage analytics. |\n", + "| Read the analytics documentation (and how to opt-out) here: |\n", + "| |\n", + "| |\n", + "+---------------------------------------------------------------------+\n", + "\n", + "What's next?\n", + "------------\n", + "- Check out the documentation: \n", + "- Get help and share ideas: \n", + "- Star us on GitHub: \n" ] } ], @@ -131,7 +131,7 @@ "metadata": {}, "outputs": [], "source": [ - "from zntrack import Node, zn\n", + "from zntrack import Node, zn, Project\n", "from random import randrange\n", "\n", "\n", @@ -168,7 +168,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "3\n" + "14\n" ] } ], @@ -193,18 +193,35 @@ "execution_count": 6, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Running DVC command: 'stage add --name RandomNumber --force ...'\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:52:47,248 (WARNING): Jupyter support is an experimental feature! Please save your notebook before running this command!\n", - "Submit issues to https://github.com/zincware/ZnTrack.\n" + "Creating 'dvc.yaml'\n", + "Adding stage 'RandomNumber' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.yaml nodes/RandomNumber/.gitignore\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ + "Jupyter support is an experimental feature! Please save your notebook before running this command!\n", + "Submit issues to https://github.com/zincware/ZnTrack.\n", "[NbConvertApp] Converting notebook 01_Intro.ipynb to script\n" ] }, @@ -212,28 +229,40 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:52:48,624 (WARNING): Running DVC command: 'stage add -n RandomNumber --force ...'\n" + "Running stage 'RandomNumber':\n", + "> zntrack run src.RandomNumber.RandomNumber --name RandomNumber\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 4609 bytes to 01_Intro.py\n" + "[NbConvertApp] Writing 4644 bytes to 01_Intro.py\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:52:50,028 (INFO): \n", - "2023-02-16 09:52:50,028 (WARNING): Running DVC command: 'repro RandomNumber'\n", - "2023-02-16 09:52:51,781 (INFO): \n" + "Generating lock file 'dvc.lock'\n", + "Updating lock file 'dvc.lock'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.lock\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n", + "Use `dvc push` to send your updates to remote storage.\n" ] } ], "source": [ - "RandomNumber(maximum=512).write_graph(run=True)" + "with Project() as project:\n", + " node = RandomNumber(maximum=512)\n", + "\n", + "project.run()" ] }, { @@ -250,7 +279,9 @@ "outputs": [ { "data": { - "text/plain": "306" + "text/plain": [ + "354" + ] }, "execution_count": 7, "metadata": {}, @@ -258,18 +289,19 @@ } ], "source": [ - "RandomNumber.load().number" + "node.load()\n", + "node.number" ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "Instead of passing parameters you can also pass a parameter file (A list of all supported files, e.g. json/yaml can be found in the documentation [DVC Params](https://dvc.org/doc/command-reference/params)).\n", "To do so you can use `zntrack.dvc.params()`." - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", @@ -300,7 +332,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.0" + "version": "3.10.9" } }, "nbformat": 4, diff --git a/examples/docs/02_Inheritance.ipynb b/examples/docs/02_Inheritance.ipynb index b303ac3c..50004f79 100644 --- a/examples/docs/02_Inheritance.ipynb +++ b/examples/docs/02_Inheritance.ipynb @@ -41,24 +41,24 @@ "name": "stdout", "output_type": "stream", "text": [ - "Initialized empty Git repository in /tmp/tmpmoibptq0/.git/\r\n", - "Initialized DVC repository.\r\n", - "\r\n", - "You can now commit the changes to git.\r\n", - "\r\n", - "+---------------------------------------------------------------------+\r\n", - "| |\r\n", - "| DVC has enabled anonymous aggregate usage analytics. |\r\n", - "| Read the analytics documentation (and how to opt-out) here: |\r\n", - "| |\r\n", - "| |\r\n", - "+---------------------------------------------------------------------+\r\n", - "\r\n", - "What's next?\r\n", - "------------\r\n", - "- Check out the documentation: \r\n", - "- Get help and share ideas: \r\n", - "- Star us on GitHub: \r\n" + "Initialized empty Git repository in /tmp/tmp4o5jiism/.git/\n", + "Initialized DVC repository.\n", + "\n", + "You can now commit the changes to git.\n", + "\n", + "+---------------------------------------------------------------------+\n", + "| |\n", + "| DVC has enabled anonymous aggregate usage analytics. |\n", + "| Read the analytics documentation (and how to opt-out) here: |\n", + "| |\n", + "| |\n", + "+---------------------------------------------------------------------+\n", + "\n", + "What's next?\n", + "------------\n", + "- Check out the documentation: \n", + "- Get help and share ideas: \n", + "- Star us on GitHub: \n" ] } ], @@ -82,11 +82,14 @@ }, "outputs": [], "source": [ - "from zntrack import Node, zn" + "import zntrack" ] }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "Let us define a ``NodeBase`` which has a single input and output. We will use this as a base class for the following Nodes.\n", "\n", @@ -94,10 +97,7 @@ "- ``MultiplyNumber``: Multiply input by a factor\n", "\n", "Both of these Nodes extend the ``NodeBase`` by additional parameters." - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", @@ -110,11 +110,11 @@ }, "outputs": [], "source": [ - "class NodeBase(Node):\n", - " node_name = \"basic_number\"\n", + "class NodeBase(zntrack.Node):\n", + " _name_ = \"basic_number\"\n", "\n", - " inputs: float = zn.params()\n", - " output: float = zn.outs()" + " inputs: float = zntrack.zn.params()\n", + " output: float = zntrack.zn.outs()" ] }, { @@ -130,14 +130,14 @@ "source": [ "class AddNumber(NodeBase):\n", " \"\"\"Shift input by an offset\"\"\"\n", - " offset: float = zn.params()\n", + " offset: float = zntrack.zn.params()\n", "\n", " def run(self):\n", " self.output = self.inputs + self.offset\n", "\n", "class MultiplyNumber(NodeBase):\n", " \"\"\"Multiply input by a factor\"\"\"\n", - " factor: float = zn.params()\n", + " factor: float = zntrack.zn.params()\n", "\n", " def run(self):\n", " self.output = self.inputs * self.factor" @@ -153,18 +153,35 @@ } }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Running DVC command: 'stage add --name basic_number --force ...'\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:16,610 (WARNING): Jupyter support is an experimental feature! Please save your notebook before running this command!\n", - "Submit issues to https://github.com/zincware/ZnTrack.\n" + "Creating 'dvc.yaml'\n", + "Adding stage 'basic_number' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.yaml nodes/basic_number/.gitignore\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ + "Jupyter support is an experimental feature! Please save your notebook before running this command!\n", + "Submit issues to https://github.com/zincware/ZnTrack.\n", "[NbConvertApp] Converting notebook 02_Inheritance.ipynb to script\n" ] }, @@ -172,29 +189,39 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:17,830 (WARNING): Running DVC command: 'stage add -n basic_number --force ...'\n" + "Running stage 'basic_number':\n", + "> zntrack run src.AddNumber.AddNumber --name basic_number\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 4620 bytes to 02_Inheritance.py\n" + "[NbConvertApp] Writing 4756 bytes to 02_Inheritance.py\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:19,084 (INFO): \n", - "2023-02-16 09:57:19,085 (WARNING): Running DVC command: 'repro basic_number'\n", - "2023-02-16 09:57:20,859 (INFO): \n" + "Generating lock file 'dvc.lock'\n", + "Updating lock file 'dvc.lock'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.lock\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n", + "Use `dvc push` to send your updates to remote storage.\n" ] } ], "source": [ - "add_number = AddNumber(inputs=10.0, offset=15.0)\n", - "add_number.write_graph(run=True)" + "with zntrack.Project() as project:\n", + " add_number = AddNumber(inputs=10.0, offset=15.0)\n", + "project.run()" ] }, { @@ -221,7 +248,9 @@ "outputs": [ { "data": { - "text/plain": "25.0" + "text/plain": [ + "25.0" + ] }, "execution_count": 7, "metadata": {}, @@ -229,7 +258,7 @@ } ], "source": [ - "NodeBase.load().output" + "NodeBase.from_rev().output" ] }, { @@ -246,9 +275,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "+--------------+ \r\n", - "| basic_number | \r\n", - "+--------------+ \r\n" + "+--------------+ \n", + "| basic_number | \n", + "+--------------+ \n" ] } ], @@ -266,6 +295,28 @@ } }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Running DVC command: 'stage add --name basic_number --force ...'\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Modifying stage 'basic_number' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" + ] + }, { "name": "stderr", "output_type": "stream", @@ -277,29 +328,38 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:22,852 (WARNING): Running DVC command: 'stage add -n basic_number --force ...'\n" + "Running stage 'basic_number':\n", + "> zntrack run src.MultiplyNumber.MultiplyNumber --name basic_number\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 4620 bytes to 02_Inheritance.py\n" + "[NbConvertApp] Writing 4756 bytes to 02_Inheritance.py\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:24,165 (INFO): \n", - "2023-02-16 09:57:24,166 (WARNING): Running DVC command: 'repro basic_number'\n", - "2023-02-16 09:57:26,249 (INFO): \n" + "Updating lock file 'dvc.lock'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.lock\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n", + "Use `dvc push` to send your updates to remote storage.\n" ] } ], "source": [ - "multiply_number = MultiplyNumber(inputs=6.0, factor=6.0)\n", - "multiply_number.write_graph(run=True)" + "with zntrack.Project() as project:\n", + " multiply_number = MultiplyNumber(inputs=6.0, factor=6.0)\n", + "project.run()" ] }, { @@ -314,7 +374,9 @@ "outputs": [ { "data": { - "text/plain": "36.0" + "text/plain": [ + "36.0" + ] }, "execution_count": 10, "metadata": {}, @@ -322,7 +384,7 @@ } ], "source": [ - "NodeBase.load().output" + "NodeBase.from_rev().output" ] }, { @@ -346,9 +408,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "+--------------+ \r\n", - "| basic_number | \r\n", - "+--------------+ \r\n" + "+--------------+ \n", + "| basic_number | \n", + "+--------------+ \n" ] } ], @@ -386,23 +448,21 @@ "source": [ "class DivideNumber(NodeBase):\n", " \"\"\"Multiply input by a factor\"\"\"\n", - " divider: float = zn.params()\n", - " _hash = zn.Hash()\n", + " divider: float = zntrack.zn.params()\n", "\n", " def run(self):\n", " self.output = self.inputs * self.divider\n", "\n", "\n", - "class Polynomial(Node):\n", - " a0: float = zn.params()\n", - " a1: float = zn.params()\n", - " _hash = zn.Hash()\n", + "class Polynomial(zntrack.Node):\n", + " a0: float = zntrack.zn.params()\n", + " a1: float = zntrack.zn.params()\n", "\n", - "class ManipulateNumber(Node):\n", - " inputs: float = zn.params()\n", - " output: float = zn.outs()\n", - " value_handler: NodeBase = zn.Nodes()\n", - " polynomial: Polynomial = zn.Nodes()\n", + "class ManipulateNumber(zntrack.Node):\n", + " inputs: float = zntrack.zn.params()\n", + " output: float = zntrack.zn.outs()\n", + " value_handler: NodeBase = zntrack.zn.nodes()\n", + " polynomial: Polynomial = zntrack.zn.nodes()\n", "\n", " def run(self):\n", " # use the passed method\n", @@ -422,89 +482,107 @@ "outputs_hidden": false } }, - "outputs": [], - "source": [ - "manipulate_number = ManipulateNumber(\n", - " inputs=10.0,\n", - " value_handler=DivideNumber(divider=3.0, inputs=None),\n", - " polynomial=Polynomial(a0=60.0, a1=10.0),\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Converting notebook 02_Inheritance.ipynb to script\n" + "Running DVC command: 'stage add --name ManipulateNumber --force ...'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:28,245 (WARNING): Running DVC command: 'stage add -n ManipulateNumber_polynomial --force ...'\n" + "Adding stage 'ManipulateNumber' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add nodes/ManipulateNumber/.gitignore dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 4620 bytes to 02_Inheritance.py\n" + "Running DVC command: 'stage add --name ManipulateNumber_polynomial --outs ...'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:29,927 (INFO): \n", - "2023-02-16 09:57:29,927 (WARNING): Running DVC command: 'repro ManipulateNumber_polynomial'\n", - "2023-02-16 09:57:31,644 (INFO): \n" + "Adding stage 'ManipulateNumber_polynomial' in 'dvc.yaml'\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Converting notebook 02_Inheritance.ipynb to script\n" + "Could not create .gitignore entry in /tmp/tmp4o5jiism/nodes/ManipulateNumber_polynomial/.gitignore. DVC will attempt to create .gitignore entry again when the stage is run.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Running DVC command: 'stage add --name ManipulateNumber_value_handler --outs ...'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:32,854 (WARNING): Running DVC command: 'stage add -n ManipulateNumber_value_handler --force ...'\n" + "Adding stage 'ManipulateNumber_value_handler' in 'dvc.yaml'\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 4620 bytes to 02_Inheritance.py\n" + "Could not create .gitignore entry in /tmp/tmp4o5jiism/nodes/ManipulateNumber_value_handler/.gitignore. DVC will attempt to create .gitignore entry again when the stage is run.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:34,087 (INFO): \n", - "2023-02-16 09:57:34,087 (WARNING): Running DVC command: 'repro ManipulateNumber_value_handler'\n", - "2023-02-16 09:57:35,826 (INFO): \n" + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ + "[NbConvertApp] Converting notebook 02_Inheritance.ipynb to script\n", + "[NbConvertApp] Writing 4756 bytes to 02_Inheritance.py\n", + "[NbConvertApp] Converting notebook 02_Inheritance.ipynb to script\n", + "[NbConvertApp] Writing 4756 bytes to 02_Inheritance.py\n", "[NbConvertApp] Converting notebook 02_Inheritance.ipynb to script\n" ] }, @@ -512,33 +590,60 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:37,037 (WARNING): Running DVC command: 'stage add -n ManipulateNumber --force ...'\n" + "Running stage 'ManipulateNumber_polynomial':\n", + "> zntrack run src.Polynomial.Polynomial --name ManipulateNumber_polynomial --hash-only\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 4620 bytes to 02_Inheritance.py\n" + "[NbConvertApp] Writing 4756 bytes to 02_Inheritance.py\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 09:57:38,393 (INFO): \n", - "2023-02-16 09:57:38,394 (WARNING): Running DVC command: 'repro ManipulateNumber'\n", - "2023-02-16 09:57:40,167 (INFO): \n" + "Updating lock file 'dvc.lock'\n", + "\n", + "Running stage 'ManipulateNumber_value_handler':\n", + "> zntrack run src.DivideNumber.DivideNumber --name ManipulateNumber_value_handler --hash-only\n", + "Updating lock file 'dvc.lock'\n", + "\n", + "Running stage 'ManipulateNumber':\n", + "> zntrack run src.ManipulateNumber.ManipulateNumber --name ManipulateNumber\n", + "Updating lock file 'dvc.lock'\n", + "\n", + "Stage 'basic_number' didn't change, skipping\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add nodes/ManipulateNumber_value_handler/.gitignore nodes/ManipulateNumber_polynomial/.gitignore dvc.lock\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n", + "Use `dvc push` to send your updates to remote storage.\n" ] } ], "source": [ - "manipulate_number.write_graph(run=True)" + "value_handler=DivideNumber(divider=3.0, inputs=None)\n", + "polynomial=Polynomial(a0=60.0, a1=10.0)\n", + "\n", + "with zntrack.Project() as project:\n", + " manipulate_number = ManipulateNumber(\n", + " inputs=10.0,\n", + " value_handler=value_handler,\n", + " polynomial=polynomial,\n", + " )\n", + "project.run()" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": { "collapsed": false, "jupyter": { @@ -547,12 +652,12 @@ }, "outputs": [], "source": [ - "manipulate_number = manipulate_number.load()" + "manipulate_number.load()" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": { "collapsed": false, "jupyter": { @@ -562,9 +667,11 @@ "outputs": [ { "data": { - "text/plain": "360.0" + "text/plain": [ + "360.0" + ] }, - "execution_count": 16, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -575,7 +682,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": { "collapsed": false, "jupyter": { @@ -587,18 +694,18 @@ "name": "stdout", "output_type": "stream", "text": [ - "+--------------+ \r\n", - "| basic_number | \r\n", - "+--------------+ \r\n", - "+-----------------------------+ +--------------------------------+\r\n", - "| ManipulateNumber_polynomial | | ManipulateNumber_value_handler |\r\n", - "+-----------------------------+ +--------------------------------+\r\n", - " **** ***** \r\n", - " **** **** \r\n", - " *** *** \r\n", - " +------------------+ \r\n", - " | ManipulateNumber | \r\n", - " +------------------+ \r\n" + "+--------------+ \n", + "| basic_number | \n", + "+--------------+ \n", + "+-----------------------------+ +--------------------------------+\n", + "| ManipulateNumber_polynomial | | ManipulateNumber_value_handler |\n", + "+-----------------------------+ +--------------------------------+\n", + " **** ***** \n", + " **** **** \n", + " *** *** \n", + " +------------------+ \n", + " | ManipulateNumber | \n", + " +------------------+ \n" ] } ], @@ -608,7 +715,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": { "collapsed": false, "jupyter": { @@ -637,7 +744,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.10.9" } }, "nbformat": 4, diff --git a/examples/docs/03_dependencies.ipynb b/examples/docs/03_dependencies.ipynb index 25afd5f9..47ca013f 100644 --- a/examples/docs/03_dependencies.ipynb +++ b/examples/docs/03_dependencies.ipynb @@ -27,7 +27,7 @@ "metadata": {}, "outputs": [], "source": [ - "from zntrack import Node, dvc, zn, config\n", + "import zntrack\n", "from random import randrange\n", "from pathlib import Path" ] @@ -39,7 +39,7 @@ "metadata": {}, "outputs": [], "source": [ - "config.nb_name = \"03_dependencies.ipynb\"" + "zntrack.config.nb_name = \"03_dependencies.ipynb\"" ] }, { @@ -67,24 +67,24 @@ "name": "stdout", "output_type": "stream", "text": [ - "Initialized empty Git repository in /tmp/tmpnpwc7foi/.git/\r\n", - "Initialized DVC repository.\r\n", - "\r\n", - "You can now commit the changes to git.\r\n", - "\r\n", - "+---------------------------------------------------------------------+\r\n", - "| |\r\n", - "| DVC has enabled anonymous aggregate usage analytics. |\r\n", - "| Read the analytics documentation (and how to opt-out) here: |\r\n", - "| |\r\n", - "| |\r\n", - "+---------------------------------------------------------------------+\r\n", - "\r\n", - "What's next?\r\n", - "------------\r\n", - "- Check out the documentation: \r\n", - "- Get help and share ideas: \r\n", - "- Star us on GitHub: \r\n" + "Initialized empty Git repository in /tmp/tmpazvjbvbl/.git/\n", + "Initialized DVC repository.\n", + "\n", + "You can now commit the changes to git.\n", + "\n", + "+---------------------------------------------------------------------+\n", + "| |\n", + "| DVC has enabled anonymous aggregate usage analytics. |\n", + "| Read the analytics documentation (and how to opt-out) here: |\n", + "| |\n", + "| |\n", + "+---------------------------------------------------------------------+\n", + "\n", + "What's next?\n", + "------------\n", + "- Check out the documentation: \n", + "- Get help and share ideas: \n", + "- Star us on GitHub: \n" ] } ], @@ -102,24 +102,25 @@ }, "outputs": [], "source": [ - "class RandomNumber(Node):\n", - " maximum = zn.params()\n", - " number = zn.outs()\n", + "class RandomNumber(zntrack.Node):\n", + " maximum = zntrack.zn.params()\n", + " number = zntrack.zn.outs()\n", "\n", " def run(self):\n", " self.number = float(randrange(self.maximum))\n", "\n", "\n", - "class ComputePower(Node):\n", - " random_number: RandomNumber = zn.deps()\n", - " number = zn.outs()\n", - " power = zn.params()\n", + "class ComputePower(zntrack.Node):\n", + " random_number: RandomNumber = zntrack.zn.deps()\n", + " number = zntrack.zn.outs()\n", + " power = zntrack.zn.params()\n", "\n", " def run(self):\n", " self.number = self.random_number.number**self.power" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "e6f37308-a942-467d-94f6-1e6edfe7b317", "metadata": {}, @@ -128,17 +129,7 @@ "This will create the following graph for us:\n", "\n", "\n", - "[![](https://mermaid.ink/img/pako:eNpVjLEKwkAQRH8lTG0KLa-wMbWI2rkWa25jAtm7cO4RJOTfPQQLu2Hem1nQRi9w6MY4tz0nq64NBbJQ6u2NcIg6ZZNTnCUR7j-0K-jMwUc9Zn38o7ref9fYQCUpD778LxSqimC9qBBciV46zqMRKKxF5Wzx8g4tnKUsG-TJs0kz8DOxwnU8vmT9ACo6PUg?type=png)](https://mermaid.live/edit#pako:eNpVjLEKwkAQRH8lTG0KLa-wMbWI2rkWa25jAtm7cO4RJOTfPQQLu2Hem1nQRi9w6MY4tz0nq64NBbJQ6u2NcIg6ZZNTnCUR7j-0K-jMwUc9Zn38o7ref9fYQCUpD778LxSqimC9qBBciV46zqMRKKxF5Wzx8g4tnKUsG-TJs0kz8DOxwnU8vmT9ACo6PUg)\n", - "\n", - "
\n", - "Note\n", - "\n", - "Instead of passing the class as an argument it is also possible to define a default dependency.\n", - "\n", - "```python\n", - "dependency: Stage = zn.deps(Stage) # or zn.deps(Stage.load())\n", - "```\n", - "
" + "[![](https://mermaid.ink/img/pako:eNpVjLEKwkAQRH8lTG0KLa-wMbWI2rkWa25jAtm7cO4RJOTfPQQLu2Hem1nQRi9w6MY4tz0nq64NBbJQ6u2NcIg6ZZNTnCUR7j-0K-jMwUc9Zn38o7ref9fYQCUpD778LxSqimC9qBBciV46zqMRKKxF5Wzx8g4tnKUsG-TJs0kz8DOxwnU8vmT9ACo6PUg?type=png)](https://mermaid.live/edit#pako:eNpVjLEKwkAQRH8lTG0KLa-wMbWI2rkWa25jAtm7cO4RJOTfPQQLu2Hem1nQRi9w6MY4tz0nq64NBbJQ6u2NcIg6ZZNTnCUR7j-0K-jMwUc9Zn38o7ref9fYQCUpD778LxSqimC9qBBciV46zqMRKKxF5Wzx8g4tnKUsG-TJs0kz8DOxwnU8vmT9ACo6PUg)" ] }, { @@ -147,79 +138,73 @@ "id": "00353190-29c3-4954-a9b5-aec358b35fa9", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-02-16 10:19:16,830 (WARNING): Jupyter support is an experimental feature! Please save your notebook before running this command!\n", - "Submit issues to https://github.com/zincware/ZnTrack.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[NbConvertApp] Converting notebook 03_dependencies.ipynb to script\n", - "/data/fzills/miniconda3/envs/zntrack/lib/python3.10/site-packages/nbformat/__init__.py:93: MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions. You may want to use `normalize()` on your notebooks before validations (available since nbformat 5.1.4). Previous versions of nbformat are fixing this issue transparently, and will stop doing so in the future.\n", - " validate(nb)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-02-16 10:19:18,182 (WARNING): Running DVC command: 'stage add -n RandomNumber --force ...'\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 6156 bytes to 03_dependencies.py\n" + "Running DVC command: 'stage add --name RandomNumber --force ...'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 10:19:19,555 (INFO): \n" + "Creating 'dvc.yaml'\n", + "Adding stage 'RandomNumber' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add nodes/RandomNumber/.gitignore dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ + "Jupyter support is an experimental feature! Please save your notebook before running this command!\n", + "Submit issues to https://github.com/zincware/ZnTrack.\n", "[NbConvertApp] Converting notebook 03_dependencies.ipynb to script\n", "/data/fzills/miniconda3/envs/zntrack/lib/python3.10/site-packages/nbformat/__init__.py:93: MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions. You may want to use `normalize()` on your notebooks before validations (available since nbformat 5.1.4). Previous versions of nbformat are fixing this issue transparently, and will stop doing so in the future.\n", - " validate(nb)\n" + " validate(nb)\n", + "[NbConvertApp] Writing 6203 bytes to 03_dependencies.py\n", + "Running DVC command: 'stage add --name ComputePower --force ...'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 10:19:20,813 (WARNING): Running DVC command: 'stage add -n ComputePower --force ...'\n" + "Adding stage 'ComputePower' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add nodes/ComputePower/.gitignore dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 6156 bytes to 03_dependencies.py\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-02-16 10:19:22,080 (INFO): \n" + "[NbConvertApp] Converting notebook 03_dependencies.ipynb to script\n", + "/data/fzills/miniconda3/envs/zntrack/lib/python3.10/site-packages/nbformat/__init__.py:93: MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions. You may want to use `normalize()` on your notebooks before validations (available since nbformat 5.1.4). Previous versions of nbformat are fixing this issue transparently, and will stop doing so in the future.\n", + " validate(nb)\n", + "[NbConvertApp] Writing 6203 bytes to 03_dependencies.py\n" ] } ], "source": [ - "random_number = RandomNumber(maximum=16)\n", - "random_number.write_graph()\n", - "ComputePower(random_number=random_number, power=2.0).write_graph()" + "with zntrack.Project() as project:\n", + " random_number = RandomNumber(maximum=16)\n", + " compute_power = ComputePower(random_number=random_number, power=2.0)\n", + "project.run(repro=False)" ] }, { @@ -232,23 +217,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running stage 'RandomNumber': \r\n", - "> zntrack run src.RandomNumber.RandomNumber --name=RandomNumber\r\n", - "Generating lock file 'dvc.lock' \r\n", - "Updating lock file 'dvc.lock'\r\n", - "\r\n", - "Running stage 'ComputePower':\r\n", - "> zntrack run src.ComputePower.ComputePower --name=ComputePower\r\n", - "Updating lock file 'dvc.lock' \r\n", - "\r\n", - "To track the changes with git, run:\r\n", - "\r\n", - "\tgit add dvc.lock\r\n", - "\r\n", - "To enable auto staging, run:\r\n", - "\r\n", - "\tdvc config core.autostage true\r\n", - "Use `dvc push` to send your updates to remote storage.\r\n" + "Running stage 'RandomNumber': \n", + "> zntrack run src.RandomNumber.RandomNumber --name RandomNumber\n", + "Generating lock file 'dvc.lock' \n", + "Updating lock file 'dvc.lock'\n", + "\n", + "Running stage 'ComputePower':\n", + "> zntrack run src.ComputePower.ComputePower --name ComputePower\n", + "Updating lock file 'dvc.lock' \n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.lock\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n", + "Use `dvc push` to send your updates to remote storage.\n" ] } ], @@ -266,14 +251,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "1.0 ^ 2.0 = 1.0\n" + "3.0 ^ 2.0 = 9.0\n" ] } ], "source": [ + "random_number.load()\n", + "compute_power.load()\n", "print(\n", - " f\"{RandomNumber.load().number} ^ {ComputePower.load().power} =\"\n", - " f\" {ComputePower.load().number}\"\n", + " f\"{random_number.number} ^ {compute_power.power} =\"\n", + " f\" {compute_power.number}\"\n", ")" ] }, @@ -313,28 +300,27 @@ "outputs": [], "source": [ "# zntrack: break\n", - "from zntrack import nwd\n", "\n", - "class WriteToFile(Node):\n", - " random_number: RandomNumber = zn.deps(RandomNumber)\n", - " file: Path = dvc.outs(nwd / \"random_number.txt\")\n", + "class WriteToFile(zntrack.Node):\n", + " random_number: RandomNumber = zntrack.zn.deps()\n", + " file: Path = zntrack.dvc.outs(zntrack.nwd / \"random_number.txt\")\n", "\n", " def run(self):\n", " self.file.write_text(str(self.random_number.number))\n", "\n", "\n", - "class PowerFromFile(Node):\n", - " file: Path = zn.deps()\n", - " number = zn.outs()\n", - " power = zn.params(2)\n", + "class PowerFromFile(zntrack.Node):\n", + " file: Path = zntrack.zn.deps()\n", + " number = zntrack.zn.outs()\n", + " power = zntrack.zn.params(2)\n", "\n", " def run(self):\n", " number = float(self.file.read_text())\n", " self.number = number**self.power\n", "\n", "\n", - "class ComparePowers(Node):\n", - " power_deps = zn.deps()\n", + "class ComparePowers(zntrack.Node):\n", + " power_deps = zntrack.zn.deps()\n", "\n", " def run(self):\n", " assert self.power_deps[0].number == self.power_deps[1].number" @@ -342,16 +328,37 @@ }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "Let us create the stages and look at the graph." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "4b3d10e8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "NodeView((UUID('fcedbb2b-2f78-4e30-9d73-8663d88f83aa'), UUID('3b6008c4-a1bb-4fdf-9ce0-389c429fe4bf'), UUID('fc4b13b3-dd53-4347-8af1-3b8c6ca75b2a'), UUID('afdb6202-8db2-4f1d-a379-4af819846aee'), UUID('908d7dd4-405d-4da4-a40f-9cb323f5f0d2'), UUID('4d70c58b-8b22-44b6-aa97-43528094e209')))" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } ], - "metadata": { - "collapsed": false - } + "source": [ + "project.nodes" + ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 17, "id": "1eefb12a-65de-4c9b-a05a-b9827915b619", "metadata": {}, "outputs": [ @@ -359,30 +366,48 @@ "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Converting notebook 03_dependencies.ipynb to script\n", - "/data/fzills/miniconda3/envs/zntrack/lib/python3.10/site-packages/nbformat/__init__.py:93: MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions. You may want to use `normalize()` on your notebooks before validations (available since nbformat 5.1.4). Previous versions of nbformat are fixing this issue transparently, and will stop doing so in the future.\n", - " validate(nb)\n" + "Running DVC command: 'stage add --name RandomNumber --force ...'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 10:19:25,194 (WARNING): Running DVC command: 'stage add -n WriteToFile --force ...'\n" + "Modifying stage 'RandomNumber' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 6156 bytes to 03_dependencies.py\n" + "[NbConvertApp] Converting notebook 03_dependencies.ipynb to script\n", + "/data/fzills/miniconda3/envs/zntrack/lib/python3.10/site-packages/nbformat/__init__.py:93: MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions. You may want to use `normalize()` on your notebooks before validations (available since nbformat 5.1.4). Previous versions of nbformat are fixing this issue transparently, and will stop doing so in the future.\n", + " validate(nb)\n", + "[NbConvertApp] Writing 6203 bytes to 03_dependencies.py\n", + "Running DVC command: 'stage add --name ComputePower --force ...'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 10:19:26,305 (INFO): \n" + "Modifying stage 'ComputePower' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { @@ -391,28 +416,50 @@ "text": [ "[NbConvertApp] Converting notebook 03_dependencies.ipynb to script\n", "/data/fzills/miniconda3/envs/zntrack/lib/python3.10/site-packages/nbformat/__init__.py:93: MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions. You may want to use `normalize()` on your notebooks before validations (available since nbformat 5.1.4). Previous versions of nbformat are fixing this issue transparently, and will stop doing so in the future.\n", - " validate(nb)\n" + " validate(nb)\n", + "[NbConvertApp] Writing 6203 bytes to 03_dependencies.py\n", + "Running DVC command: 'stage add --name WriteToFile --force ...'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 10:19:27,624 (WARNING): Running DVC command: 'stage add -n PowerFromFile --force ...'\n" + "Adding stage 'WriteToFile' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.yaml nodes/WriteToFile/.gitignore\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 6156 bytes to 03_dependencies.py\n" + "[NbConvertApp] Converting notebook 03_dependencies.ipynb to script\n", + "/data/fzills/miniconda3/envs/zntrack/lib/python3.10/site-packages/nbformat/__init__.py:93: MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions. You may want to use `normalize()` on your notebooks before validations (available since nbformat 5.1.4). Previous versions of nbformat are fixing this issue transparently, and will stop doing so in the future.\n", + " validate(nb)\n", + "[NbConvertApp] Writing 6203 bytes to 03_dependencies.py\n", + "Running DVC command: 'stage add --name PowerFromFile --force ...'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 10:19:28,975 (INFO): \n" + "Adding stage 'PowerFromFile' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add nodes/PowerFromFile/.gitignore dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { @@ -421,127 +468,138 @@ "text": [ "[NbConvertApp] Converting notebook 03_dependencies.ipynb to script\n", "/data/fzills/miniconda3/envs/zntrack/lib/python3.10/site-packages/nbformat/__init__.py:93: MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions. You may want to use `normalize()` on your notebooks before validations (available since nbformat 5.1.4). Previous versions of nbformat are fixing this issue transparently, and will stop doing so in the future.\n", - " validate(nb)\n" + " validate(nb)\n", + "[NbConvertApp] Writing 6203 bytes to 03_dependencies.py\n", + "Running DVC command: 'stage add --name ComparePowers --force ...'\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 10:19:30,262 (WARNING): Running DVC command: 'stage add -n ComparePowers --force ...'\n" + "Adding stage 'ComparePowers' in 'dvc.yaml'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.yaml\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[NbConvertApp] Writing 6156 bytes to 03_dependencies.py\n" + "[NbConvertApp] Converting notebook 03_dependencies.ipynb to script\n", + "/data/fzills/miniconda3/envs/zntrack/lib/python3.10/site-packages/nbformat/__init__.py:93: MissingIDFieldWarning: Code cell is missing an id field, this will become a hard error in future nbformat versions. You may want to use `normalize()` on your notebooks before validations (available since nbformat 5.1.4). Previous versions of nbformat are fixing this issue transparently, and will stop doing so in the future.\n", + " validate(nb)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2023-02-16 10:19:31,524 (INFO): \n" + "Stage 'RandomNumber' didn't change, skipping\n", + "Running stage 'WriteToFile':\n", + "> zntrack run src.WriteToFile.WriteToFile --name WriteToFile\n" ] - } - ], - "source": [ - "write_to_file = WriteToFile()\n", - "write_to_file.write_graph()\n", - "power_from_file = PowerFromFile(file=write_to_file.file)\n", - "power_from_file.write_graph()\n", - "ComparePowers(power_deps=[power_from_file, ComputePower.load()]).write_graph()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "883099de-6823-4000-9080-6f57b61a9bf9", - "metadata": {}, - "outputs": [ + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[NbConvertApp] Writing 6203 bytes to 03_dependencies.py\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - " +--------------+ \r\n", - " | RandomNumber | \r\n", - " +--------------+ \r\n", - " ** *** \r\n", - " *** *** \r\n", - " ** ** \r\n", - " +-------------+ ** \r\n", - " | WriteToFile | * \r\n", - " +-------------+ * \r\n", - " * * \r\n", - " * * \r\n", - " * * \r\n", - "+---------------+ +--------------+ \r\n", - "| PowerFromFile | | ComputePower | \r\n", - "+---------------+ +--------------+ \r\n", - " ** *** \r\n", - " *** ** \r\n", - " ** ** \r\n", - " +---------------+ \r\n", - " | ComparePowers | \r\n", - " +---------------+ \r\n" + "Updating lock file 'dvc.lock'\n", + "\n", + "Running stage 'PowerFromFile':\n", + "> zntrack run src.PowerFromFile.PowerFromFile --name PowerFromFile\n", + "Updating lock file 'dvc.lock'\n", + "\n", + "Stage 'ComputePower' didn't change, skipping\n", + "Running stage 'ComparePowers':\n", + "> zntrack run src.ComparePowers.ComparePowers --name ComparePowers\n", + "Updating lock file 'dvc.lock'\n", + "\n", + "To track the changes with git, run:\n", + "\n", + "\tgit add dvc.lock\n", + "\n", + "To enable auto staging, run:\n", + "\n", + "\tdvc config core.autostage true\n", + "Use `dvc push` to send your updates to remote storage.\n" ] } ], "source": [ - "!dvc dag" + "with zntrack.Project() as project:\n", + " random_number = RandomNumber(maximum=16)\n", + " compute_power = ComputePower(random_number=random_number, power=2.0)\n", + "\n", + " write_to_file = WriteToFile(random_number=random_number)\n", + " power_from_file = PowerFromFile(file=write_to_file.file)\n", + " compare_powerts = ComparePowers(power_deps=[power_from_file, compute_power])\n", + "project.run()" ] }, { "cell_type": "code", - "execution_count": 12, - "id": "20944b0c-9c56-49fa-bff8-e77886d4e50f", + "execution_count": 18, + "id": "883099de-6823-4000-9080-6f57b61a9bf9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Stage 'RandomNumber' didn't change, skipping \r\n", - "Stage 'ComputePower' didn't change, skipping\r\n", - "Running stage 'WriteToFile':\r\n", - "> zntrack run src.WriteToFile.WriteToFile --name=WriteToFile\r\n", - "Updating lock file 'dvc.lock' \r\n", - "\r\n", - "Running stage 'PowerFromFile':\r\n", - "> zntrack run src.PowerFromFile.PowerFromFile --name=PowerFromFile\r\n", - "Updating lock file 'dvc.lock' \r\n", - "\r\n", - "Running stage 'ComparePowers':\r\n", - "> zntrack run src.ComparePowers.ComparePowers --name=ComparePowers\r\n", - "Updating lock file 'dvc.lock' \r\n", - "\r\n", - "To track the changes with git, run:\r\n", - "\r\n", - "\tgit add dvc.lock\r\n", - "\r\n", - "To enable auto staging, run:\r\n", - "\r\n", - "\tdvc config core.autostage true\r\n", - "Use `dvc push` to send your updates to remote storage.\r\n" + " +--------------+ \n", + " | RandomNumber | \n", + " +--------------+ \n", + " ** *** \n", + " *** *** \n", + " ** ** \n", + " +-------------+ ** \n", + " | WriteToFile | * \n", + " +-------------+ * \n", + " * * \n", + " * * \n", + " * * \n", + "+---------------+ +--------------+ \n", + "| PowerFromFile | | ComputePower | \n", + "+---------------+ +--------------+ \n", + " ** *** \n", + " *** ** \n", + " ** ** \n", + " +---------------+ \n", + " | ComparePowers | \n", + " +---------------+ \n" ] } ], "source": [ - "!dvc repro" + "!dvc dag" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 19, + "metadata": { + "collapsed": false + }, "outputs": [], "source": [ "# to verify we can also run the method manually\n", - "ComparePowers.load().run()" - ], - "metadata": { - "collapsed": false - } + "compare_powerts.load()\n", + "compare_powerts.run()" + ] }, { "cell_type": "markdown", @@ -553,13 +611,48 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 20, "id": "32e1e993-ff99-4eb6-97db-ffc3ce834ca5", "metadata": {}, "outputs": [ { "data": { - "text/plain": "stages:\n RandomNumber:\n cmd: zntrack run src.RandomNumber.RandomNumber --name=RandomNumber\n deps:\n - src/RandomNumber.py\n params:\n - RandomNumber\n outs:\n - nodes/RandomNumber/outs.json\n ComputePower:\n cmd: zntrack run src.ComputePower.ComputePower --name=ComputePower\n deps:\n - nodes/RandomNumber/outs.json\n - src/ComputePower.py\n params:\n - ComputePower\n outs:\n - nodes/ComputePower/outs.json\n WriteToFile:\n cmd: zntrack run src.WriteToFile.WriteToFile --name=WriteToFile\n deps:\n - nodes/RandomNumber/outs.json\n - src/WriteToFile.py\n outs:\n - nodes/WriteToFile/random_number.txt\n PowerFromFile:\n cmd: zntrack run src.PowerFromFile.PowerFromFile --name=PowerFromFile\n deps:\n - nodes/WriteToFile/random_number.txt\n - src/PowerFromFile.py\n params:\n - PowerFromFile\n outs:\n - nodes/PowerFromFile/outs.json\n ComparePowers:\n cmd: zntrack run src.ComparePowers.ComparePowers --name=ComparePowers\n deps:\n - nodes/ComputePower/outs.json\n - nodes/PowerFromFile/outs.json\n - src/ComparePowers.py\n" + "text/plain": [ + "stages:\n", + " RandomNumber:\n", + " cmd: zntrack run src.RandomNumber.RandomNumber --name RandomNumber\n", + " params:\n", + " - RandomNumber\n", + " outs:\n", + " - nodes/RandomNumber/number.json\n", + " ComputePower:\n", + " cmd: zntrack run src.ComputePower.ComputePower --name ComputePower\n", + " deps:\n", + " - nodes/RandomNumber/number.json\n", + " params:\n", + " - ComputePower\n", + " outs:\n", + " - nodes/ComputePower/number.json\n", + " WriteToFile:\n", + " cmd: zntrack run src.WriteToFile.WriteToFile --name WriteToFile\n", + " deps:\n", + " - nodes/RandomNumber/number.json\n", + " outs:\n", + " - nodes/WriteToFile/random_number.txt\n", + " PowerFromFile:\n", + " cmd: zntrack run src.PowerFromFile.PowerFromFile --name PowerFromFile\n", + " deps:\n", + " - nodes/WriteToFile/random_number.txt\n", + " params:\n", + " - PowerFromFile\n", + " outs:\n", + " - nodes/PowerFromFile/number.json\n", + " ComparePowers:\n", + " cmd: zntrack run src.ComparePowers.ComparePowers --name ComparePowers\n", + " deps:\n", + " - nodes/ComputePower/number.json\n", + " - nodes/PowerFromFile/number.json\n" + ] }, "metadata": {}, "output_type": "display_data" @@ -573,6 +666,9 @@ }, { "cell_type": "markdown", + "metadata": { + "collapsed": false + }, "source": [ "## Node attributes as dependencies\n", "\n", @@ -580,44 +676,81 @@ "This can be used for all `dvc.