Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: generate mkdocs graphs with multiple dependencies #86

Merged
merged 5 commits into from
Nov 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions niceml/mkdocs/mdgraph.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,20 @@
"""Module for generating a graph in mkdocs"""
from dagster import JobDefinition
from dagster import JobDefinition, DependencyDefinition, MultiDependencyDefinition


def get_graph_md(job: JobDefinition) -> str:
"""Creates a graph as str with material for mkdocs"""
deps = job.graph.dependencies
graph_str = ""
for key, value in deps.items():
for _, val2 in value.items():
graph_str += f" {val2.node} --> {key.name};\n"
for node, node_dependencies in deps.items():
for _, dependencies in node_dependencies.items():
if isinstance(dependencies, DependencyDefinition):
graph_str += f" {dependencies.node} --> {node.name};\n"
elif isinstance(dependencies, MultiDependencyDefinition):
for dependency in dependencies.dependencies:
graph_str += f" {dependency.node} --> {node.name};\n"
else:
raise AttributeError("'dependencies' is not of expected type.")
if len(graph_str) == 0:
return ""

Expand Down
8 changes: 3 additions & 5 deletions niceml/mkdocs/mdjob.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
"""Module for generating mkdocs str for jobs"""
from typing import List

from dagster.core.definitions import NodeDefinition

from niceml.mkdocs.mdgraph import get_graph_md
from niceml.mkdocs.mdop import get_md_op
from dagster import JobDefinition
from dagster import JobDefinition, OpDefinition


def get_job_md(job: JobDefinition, include_graph: bool = True) -> str:
Expand All @@ -17,13 +15,13 @@ def get_job_md(job: JobDefinition, include_graph: bool = True) -> str:
graph_md = get_graph_md(job)
if len(graph_md) > 0:
job_md += graph_md + "\n\n"
op_list: List[NodeDefinition] = get_ops_from_job(job)
op_list: List[OpDefinition] = get_ops_from_job(job)
for cur_op in op_list:
job_md += get_md_op(cur_op)

return job_md


def get_ops_from_job(job: JobDefinition) -> List[NodeDefinition]:
def get_ops_from_job(job: JobDefinition) -> List[OpDefinition]:
"""Returns all ops from job"""
return job.all_node_defs
8 changes: 3 additions & 5 deletions niceml/mkdocs/mdop.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
"""Module for generating markdown strings for dagster ops"""
from typing import Dict, List

from dagster._core.definitions import NodeDefinition

from niceml.mkdocs.mdtable import get_md_table
from dagster import Field
from dagster import Field, OpDefinition


def get_md_op(op_def: NodeDefinition) -> str:
def get_md_op(op_def: OpDefinition) -> str:
"""generates markdown strings for dagster ops"""
col_widths: List[int] = [80, 120]
op_fields = get_op_fields(op_def)
Expand All @@ -24,7 +22,7 @@ def get_md_op(op_def: NodeDefinition) -> str:
return cur_md


def get_op_fields(op_def: NodeDefinition) -> Dict[str, Field]:
def get_op_fields(op_def: OpDefinition) -> Dict[str, Field]:
"""returns fields from OpDefinition"""
try:
return op_def.config_schema.config_type.fields
Expand Down