From 1298909bb8994adc0845c3bf5b622996c08a95a7 Mon Sep 17 00:00:00 2001 From: Michael Kovalsky Date: Sun, 16 Jun 2024 11:20:58 +0300 Subject: [PATCH] * moved tom to its own folder. * updated notebooks for pypi * updated readme for pypi * fixed tom functions missing TOM import --- README.md | 197 +------------ notebooks/Migration to Direct Lake.ipynb | 2 +- notebooks/Model Optimization.ipynb | 2 +- notebooks/Query Scale Out.ipynb | 2 +- notebooks/Tabular Object Model.ipynb | 2 +- src/sempy_labs/__init__.py | 44 +-- src/sempy_labs/_generate_semantic_model.py | 3 +- src/sempy_labs/_helper_functions.py | 9 +- src/sempy_labs/_list_functions.py | 263 +++++++++++++++++- src/sempy_labs/_model_auto_build.py | 2 +- src/sempy_labs/_one_lake_integration.py | 1 + src/sempy_labs/_translations.py | 2 +- .../directlake/_directlake_schema_sync.py | 68 ++--- .../directlake/_get_directlake_lakehouse.py | 1 - .../_list_directlake_model_calc_tables.py | 47 ++-- .../_show_unsupported_directlake_objects.py | 3 +- ...e_directlake_model_lakehouse_connection.py | 2 +- .../_update_directlake_partition_entity.py | 3 +- src/sempy_labs/lakehouse/__init__.py | 6 +- .../lakehouse/_get_lakehouse_tables.py | 1 + src/sempy_labs/lakehouse/_shortcuts.py | 93 ------- src/sempy_labs/migration/__init__.py | 6 +- .../_migrate_calctables_to_lakehouse.py | 2 +- .../_migrate_calctables_to_semantic_model.py | 2 +- ...migrate_model_objects_to_semantic_model.py | 2 +- ...igrate_tables_columns_to_semantic_model.py | 2 +- .../migration/_migration_validation.py | 165 +---------- .../migration/_refresh_calc_tables.py | 2 +- src/sempy_labs/report/__init__.py | 4 +- src/sempy_labs/tom/__init__.py | 6 + src/sempy_labs/{_tom.py => tom/model.py} | 38 +-- 31 files changed, 403 insertions(+), 579 deletions(-) create mode 100644 src/sempy_labs/tom/__init__.py rename src/sempy_labs/{_tom.py => tom/model.py} (99%) diff --git a/README.md b/README.md index 1c5c9649..0335f155 100644 --- a/README.md +++ b/README.md @@ -14,15 +14,19 @@ If you encounter any issues, please [raise a bug](https://github.com/microsoft/s If you have ideas for new features/functions, please [request a feature](https://github.com/microsoft/semantic-link-labs/issues/new?assignees=&labels=&projects=&template=feature_request.md&title=). -## Install the .whl file in a Fabric notebook +## [Function documentation](https://semantic-link-labs.readthedocs.io/en/latest/sempy_labs.html) + +## Install the library in a Fabric notebook ```python -%pip install "https://raw.githubusercontent.com/microsoft/semantic-link-labs/main/semantic-link-labs-0.4.1-py3-none-any.whl" +%pip install semantic-link-labs ``` ## Once installed, run this code to import the library into your notebook ```python -import semantic-link-labs as labs -from semantic-link-labs.TOM import connect_semantic_model +import sempy_labs as labs +from sempy_labs import migration, report, directlake +from sempy_labs import lakehouse as lake +from sempy_labs._tom import connect_semantic_model ``` ## Load semantic-link-labs into a custom [Fabric environment](https://learn.microsoft.com/fabric/data-engineering/create-and-use-environment) @@ -35,189 +39,16 @@ An even better way to ensure the semantic-link-labs library is available in your 4. Name your environment, click 'Create' #### Add semantic-link-labs as a library to the environment -1. Download the [latest](https://github.com/microsoft/semantic-link-labs/raw/main/semantic-link-labs-0.4.1-py3-none-any.whl) semantic-link-labs library -2. Within 'Custom Libraries', click 'upload' -3. Upload the .whl file which was downloaded in step 1 -4. Click 'Save' at the top right of the screen -5. Click 'Publish' at the top right of the screen -6. Click 'Publish All' +1. Within 'Public libraries', click 'Add from PyPI' +2. Enter 'semantic-link-labs'. +3. Click 'Save' at the top right of the screen +4. Click 'Publish' at the top right of the screen +5. Click 'Publish All' #### Update your notebook to use the new environment (*must wait for the environment to finish publishing*) 1. Navigate to your Notebook 2. Select your newly created environment within the 'Environment' drop down in the navigation bar at the top of the notebook -# Function Categories - -### Semantic Model -* [clear_cache](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#clear_cache) -* [create_semantic_model_from_bim](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#create_semantic_model_from_bim) -* [get_semantic_model_bim](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_semantic_model_bim) -* [get_measure_dependencies](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_measure_dependencies) -* [get_model_calc_dependencies](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_model_calc_dependencies) -* [measure_dependency_tree](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#measure_dependency_tree) -* [refresh_semantic_model](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#refresh_semantic_model) -* [cancel_dataset_refresh](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#cancel_dataset_refresh) -* [run_dax](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#run_dax) -* [get_object_level_security](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_object_level_security) -* [translate_semantic_model](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#translate_semantic_model) -* [list_semantic_model_objects](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#list_semantic_model_objects) - -### Report -* [report_rebind](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#report_rebind) -* [report_rebind_all](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#report_rebind_all) -* [create_report_from_reportjson](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#create_report_from_reportjson) -* [get_report_json](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_report_json) -* [export_report](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#export_report) -* [clone_report](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#clone_report) -* [list_dashboards](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#list_dashboards) -* [launch_report](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#launch_report) -* [generate_embedded_filter](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#generate_embedded_filter) - -### Model Optimization -* [vertipaq_analyzer](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#vertipaq_analyzer) -* [import_vertipaq_analyzer](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#import_vertipaq_analyzer) -* [run_model_bpa](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#run_model_bpa) -* [model_bpa_rules](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#model_bpa_rules) - -### Direct Lake Migration -* [create_pqt_file](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#create_pqt_file) -* [create_blank_semantic_model](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#create_blank_semantic_model) -* [migrate_field_parameters](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#migrate_field_parameters) -* [migrate_tables_columns_to_semantic_model](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#migrate_tables_columns_to_semantic_model) -* [migrate_calc_tables_to_semantic_model](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#migrate_calc_tables_to_semantic_model) -* [migrate_model_objects_to_semantic_model](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#migrate_model_objects_to_semantic_model) -* [migrate_calc_tables_to_lakehouse](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#migrate_calc_tables_to_lakehouse) -* [refresh_calc_tables](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#refresh_calc_tables) -* [show_unsupported_direct_lake_objects](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#show_unsupported_direct_lake_objects) -* [update_direct_lake_partition_entity](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#update_direct_lake_partition_entity) -* [update_direct_lake_model_lakehouse_connection](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#update_direct_lake_model_lakehouse_connection) -* [migration_validation](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#migration_validation) - -### Direct Lake -* [check_fallback_reason](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#check_fallback_reason) -* [control_fallback](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#control_fallback) -* [direct_lake_schema_compare](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#direct_lake_schema_compare) -* [direct_lake_schema_sync](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#direct_lake_schema_sync) -* [get_direct_lake_lakehouse](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_direct_lake_lakehouse) -* [get_directlake_guardrails_for_sku](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_directlake_guardrails_for_sku) -* [get_direct_lake_guardrails](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_direct_lake_guardrails) -* [get_shared_expression](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_shared_expression) -* [get_direct_lake_sql_endpoint](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_direct_lake_sql_endpoint) -* [get_sku_size](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_sku_size) -* [list_direct_lake_model_calc_tables](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#list_direct_lake_model_calc_tables) -* [warm_direct_lake_cache_perspective](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#warm_direct_lake_cache_perspective) -* [warm_direct_lake_cache_isresident](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#warm_direct_lake_cache_isresident) - -### Lakehouse -* [get_lakehouse_tables](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_lakehouse_tables) -* [get_lakehouse_columns](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_lakehouse_columns) -* [list_lakehouses](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#list_lakehouses) -* [export_model_to_onelake](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#export_model_to_onelake) -* [create_shortcut_onelake](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#create_shortcut_onelake) -* [delete_shortcut](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#delete_shortcut) -* [list_shortcuts](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#list_shortcuts) -* [optimize_lakehouse_tables](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#optimize_lakehouse_tables) -* [create_warehouse](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#create_warehouse) -* [update_item](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#update_item) -* [list_dataflow_storage_accounts](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#list_dataflow_storage_accounts) -* [list_warehouses](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#list_warehouses) -* [save_as_delta_table](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#save_as_delta_table) - -### Helper Functions -* [resolve_dataset_id](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#resolve_dataset_id) -* [resolve_dataset_name](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#resolve_dataset_name) -* [resolve_lakehouse_id](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#resolve_lakehouse_id) -* [resolve_lakehouse_name](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#resolve_lakehouse_name) -* [resolve_report_id](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#resolve_report_id) -* [resolve_report_name](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-files#resolve_report_name) - -### [Tabular Object Model](https://learn.microsoft.com/analysis-services/tom/introduction-to-the-tabular-object-model-tom-in-analysis-services-amo?view=asallproducts-allversions) ([TOM](https://learn.microsoft.com/dotnet/api/microsoft.analysisservices.tabular.model?view=analysisservices-dotnet)) -#### 'All' functions for non-parent objects within TOM -* [all_columns](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#all_columns) -* [all_measures](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#all_measures) -* [all_partitions](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#all_partitions) -* [all_hierarchies](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#all_hierarchies) -* [all_levels](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#all_levels) -* [all_calculation_items](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#all_calculation_items) -* [all_rls](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#all_rls) - -#### 'Add' functions -* [add_calculated_column](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_calculated_column) -* [add_calculated_table](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_calculated_table) -* [add_calculated_table_column](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_calculated_table_column) -* [add_calculation_group](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_calculation_group) -* [add_calculation_item](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_calculation_item) -* [add_data_column](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_data_column) -* [add_entity_partition](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_entity_partition) -* [add_expression](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_expression) -* [add_field_parameter](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_field_parameter) -* [add_hierarchy](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_hierarchy) -* [add_m_partition](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_m_partition) -* [add_measure](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_measure) -* [add_perspective](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_perspective) -* [add_relationship](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_relationship) -* [add_role](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_role) -* [add_table](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_table) -* [add_translation](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_translation) - -#### 'Set' functions -* [set_direct_lake_behavior](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#set_direct_lake_behavior) -* [set_is_available_in_mdx](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#set_is_available_in_mdx) -* [set_ols](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#set_ols) -* [set_rls](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_rls) -* [set_summarize_by](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#set_summarize_by) -* [set_translation](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#set_translation) - -#### 'Remove' functions -* [remove_object](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#remove_object) -* [remove_translation](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#remove_translation) - -#### 'Used-in' and dependency functions -* [used_in_relationships](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#used_in_relationships) -* [used_in_hierarchies](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#used_in_hierarchies) -* [used_in_levels](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#used_in_levels) -* [used_in_sort_by](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#used_in_sort_by) -* [used_in_rls](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#used_in_rls) -* [used_in_calc_item](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#used_in_calc_item) -* [depends_on](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#depends_on) -* [referenced_by](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#referenced_by) -* [fully_qualified_measures](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#fully_qualified_measures) -* [unqualified_columns](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#unqualified_columns) - -#### Vertipaq Analyzer data functions -* [remove_vertipaq_annotations](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#remove_vertipaq_annotations) -* [set_vertipaq_annotations](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#set_vertipaq_annotations) -* [row_count](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#row_count) -* [used_size](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#used_size) -* [data_size](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#data_size) -* [dictionary_size](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#dictionary_size) -* [total_size](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#total_size) -* [cardinality](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#cardinality) - -#### Perspectives -* [in_perspective](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#in_perspective) -* [add_to_perspective](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#add_to_perspective) -* [remove_from_perspective](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#remove_from_perspective) - -#### Annotations -* [get_annotations](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_annotations) -* [set_annotation](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#set_annotation) -* [get_annotation_value](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_annotation_value) -* [remove_annotation](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#remove_annotation) -* [clear_annotations](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#clear_annotations) - -#### Extended Properties -* [get_extended_properties](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_extended_properties) -* [set_extended_property](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#set_extended_property) -* [get_extended_property_value](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#get_extended_property_value) -* [remove_extended_property](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#remove_extended_property) -* [clear_extended_properties](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#clear_extended_properties) - -#### Misc -* [is_direct_lake](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#is_direct_lake) -* [is_field_parameter](https://github.com/microsoft/semantic-link-labs?tab=readme-ov-file#is_field_parameter) - - --- ## Direct Lake migration @@ -235,7 +66,7 @@ Check out my [blog post](https://www.elegantbi.com/post/direct-lake-migration) o ### Instructions -1. Download this [notebook](https://github.com/microsoft/semantic-link-labs/blob/main/Migration%20to%20Direct%20Lake.ipynb). **Use version 0.2.1 or higher only.** +1. Download this [notebook](https://github.com/microsoft/semantic-link-labs/blob/main/notebooks/Migration%20to%20Direct%20Lake.ipynb). 2. Make sure you are in the ['Data Engineering' persona](https://learn.microsoft.com/fabric/get-started/microsoft-fabric-overview#components-of-microsoft-fabric). Click the icon at the bottom left corner of your Workspace screen and select 'Data Engineering' 3. In your workspace, select 'New -> Import notebook' and import the notebook from step 1. 4. [Add your lakehouse](https://learn.microsoft.com/fabric/data-engineering/lakehouse-notebook-explore#add-or-remove-a-lakehouse) to your Fabric notebook diff --git a/notebooks/Migration to Direct Lake.ipynb b/notebooks/Migration to Direct Lake.ipynb index 7a32e723..5afe11ef 100644 --- a/notebooks/Migration to Direct Lake.ipynb +++ b/notebooks/Migration to Direct Lake.ipynb @@ -1 +1 @@ -{"cells":[{"cell_type":"markdown","id":"5c27dfd1-4fe0-4a97-92e6-ddf78889aa93","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Install the latest .whl package\n","\n","Check [here](https://github.com/microsoft/semantic-link-labs) to see the latest version.\n","\n","Check [here](https://github.com/microsoft/semantic-link-labs/releases) for the library's release notes."]},{"cell_type":"code","execution_count":null,"id":"d5cae9db-cef9-48a8-a351-9c5fcc99645c","metadata":{"jupyter":{"outputs_hidden":true,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["%pip install \"https://raw.githubusercontent.com/microsoft/semantic-link-labs/main/semantic-link-labs-0.4.1-py3-none-any.whl\""]},{"cell_type":"markdown","id":"969a29bf","metadata":{},"source":["### Import the library and set initial parameters"]},{"cell_type":"code","execution_count":null,"id":"29c923f8","metadata":{},"outputs":[],"source":["import sempy_labs as labs\n","from sempy_labs import migration, report, directlake\n","\n","dataset_name = '' #Enter the import/DQ semantic model name\n","workspace_name = None #Enter the workspace of the import/DQ semantic model. It set to none it will use the current workspace.\n","new_dataset_name = '' #Enter the new Direct Lake semantic model name\n","new_dataset_workspace_name = None #Enter the workspace where the Direct Lake model will be created. If set to None it will use the current workspace.\n","lakehouse_name = None #Enter the lakehouse to be used for the Direct Lake model. If set to None it will use the lakehouse attached to the notebook.\n","lakehouse_workspace_name = None #Enter the lakehouse workspace. If set to None it will use the new_dataset_workspace_name."]},{"cell_type":"markdown","id":"5a3fe6e8-b8aa-4447-812b-7931831e07fe","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Create the [Power Query Template](https://learn.microsoft.com/power-query/power-query-template) file\n","\n","This encapsulates all of the semantic model's Power Query logic into a single file."]},{"cell_type":"code","execution_count":null,"id":"cde43b47-4ecc-46ae-9125-9674819c7eab","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["migration.create_pqt_file(dataset = dataset_name, workspace = workspace_name)"]},{"cell_type":"markdown","id":"bf945d07-544c-4934-b7a6-cfdb90ca725e","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Import the Power Query Template to Dataflows Gen2\n","\n","- Open the [OneLake file explorer](https://www.microsoft.com/download/details.aspx?id=105222) and sync your files (right click -> Sync from OneLake)\n","\n","- Navigate to your lakehouse. From this window, create a new Dataflows Gen2 and import the Power Query Template file from OneLake (OneLake -> Workspace -> Lakehouse -> Files...), and publish the Dataflows Gen2.\n","\n","
\n","Important!: Make sure to create the Dataflows Gen2 from within the lakehouse window. That will ensure that all the tables automatically map to that lakehouse as the destination. Otherwise, you will have to manually map each table to its destination individually.\n","
"]},{"cell_type":"markdown","id":"9975db7d","metadata":{},"source":["### Create the Direct Lake model based on the import/DQ semantic model\n","\n","Calculated columns are not migrated to the Direct Lake model as they are not supported in Direct Lake mode."]},{"cell_type":"code","execution_count":null,"id":"0a3616b5-566e-414e-a225-fb850d6418dc","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["import time\n","labs.create_blank_semantic_model(dataset = new_dataset_name, workspace = new_dataset_workspace_name)\n","\n","time.sleep(2)\n","\n","migration.migrate_calc_tables_to_lakehouse(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name,\n"," workspace = workspace_name,\n"," new_dataset_workspace = new_dataset_workspace_name,\n"," lakehouse = lakehouse_name,\n"," lakehouse_workspace = lakehouse_workspace_name)\n","migration.migrate_tables_columns_to_semantic_model(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name,\n"," workspace = workspace_name,\n"," new_dataset_workspace = new_dataset_workspace_name,\n"," lakehouse = lakehouse_name,\n"," lakehouse_workspace = lakehouse_workspace_name)\n","migration.migrate_calc_tables_to_semantic_model(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name,\n"," workspace = workspace_name,\n"," new_dataset_workspace = new_dataset_workspace_name,\n"," lakehouse = lakehouse_name,\n"," lakehouse_workspace = lakehouse_workspace_name)\n","migration.migrate_model_objects_to_semantic_model(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name,\n"," workspace = workspace_name,\n"," new_dataset_workspace = new_dataset_workspace_name)\n","migration.migrate_field_parameters(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name,\n"," workspace = workspace_name,\n"," new_dataset_workspace = new_dataset_workspace_name)\n","time.sleep(2)\n","migration.refresh_semantic_model(dataset = new_dataset_name, workspace = new_dataset_workspace_name)\n","migration.refresh_calc_tables(dataset = new_dataset_name, workspace = new_dataset_workspace_name)\n","migration.refresh_semantic_model(dataset = new_dataset_name, workspace = new_dataset_workspace_name)"]},{"cell_type":"markdown","id":"bb98bb13","metadata":{},"source":["### Show migrated/unmigrated objects"]},{"cell_type":"code","execution_count":null,"id":"5db2f22c","metadata":{},"outputs":[],"source":["migration.migration_validation(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name, \n"," workspace = workspace_name, \n"," new_dataset_workspace = new_dataset_workspace_name)"]},{"cell_type":"markdown","id":"fa244e9d-87c2-4a66-a7e0-be539a0ac7de","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Rebind all reports using the old semantic model to the new Direct Lake semantic model"]},{"cell_type":"code","execution_count":null,"id":"d4e867cc","metadata":{},"outputs":[],"source":["report.report_rebind_all(\n"," dataset = dataset_name,\n"," dataset_workspace = workspace_name,\n"," new_dataset = new_dataset_name,\n"," new_dataset_workpace = new_dataset_workspace_name,\n"," report_workspace = workspace_name)"]},{"cell_type":"markdown","id":"3365d20d","metadata":{},"source":["### Rebind reports one-by-one (optional)"]},{"cell_type":"code","execution_count":null,"id":"056b7180-d7ac-492c-87e7-ac7d0e4bb929","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["report_name = '' # Enter report name which you want to rebind to the new Direct Lake model\n","\n","report.report_rebind(\n"," report = report_name,\n"," dataset = new_dataset_name,\n"," report_workspace=workspace_name,\n"," dataset_workspace = new_dataset_workspace_name)"]},{"cell_type":"markdown","id":"526f2327","metadata":{},"source":["### Show unsupported objects"]},{"cell_type":"code","execution_count":null,"id":"a47376d7","metadata":{},"outputs":[],"source":["dfT, dfC, dfR = directlake.show_unsupported_direct_lake_objects(dataset = dataset_name, workspace = workspace_name)\n","\n","print('Calculated Tables are not supported...')\n","display(dfT)\n","print(\"Learn more about Direct Lake limitations here: https://learn.microsoft.com/power-bi/enterprise/directlake-overview#known-issues-and-limitations\")\n","print('Calculated columns are not supported. Columns of binary data type are not supported.')\n","display(dfC)\n","print('Columns used for relationship cannot be of data type datetime and they also must be of the same data type.')\n","display(dfR)"]},{"cell_type":"markdown","id":"ed08ba4c","metadata":{},"source":["### Schema check between semantic model tables/columns and lakehouse tables/columns\n","\n","This will list any tables/columns which are in the new semantic model but do not exist in the lakehouse"]},{"cell_type":"code","execution_count":null,"id":"03889ba4","metadata":{},"outputs":[],"source":["directlake.direct_lake_schema_compare(dataset = new_dataset_name, workspace = new_dataset_workspace_name)"]},{"cell_type":"markdown","id":"2229963b","metadata":{},"source":["### Show calculated tables which have been migrated to the Direct Lake semantic model as regular tables"]},{"cell_type":"code","execution_count":null,"id":"dd537d90","metadata":{},"outputs":[],"source":["directlake.list_direct_lake_model_calc_tables(dataset = new_dataset_name, workspace = new_dataset_workspace_name)"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.3"},"microsoft":{"language":"python"},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default"},"synapse_widget":{"state":{},"version":"0.1"},"widgets":{}},"nbformat":4,"nbformat_minor":5} +{"cells":[{"cell_type":"markdown","id":"5c27dfd1-4fe0-4a97-92e6-ddf78889aa93","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Install the latest .whl package\n","\n","Check [here](https://pypi.org/project/semantic-link-labs/) to see the latest version."]},{"cell_type":"code","execution_count":null,"id":"d5cae9db-cef9-48a8-a351-9c5fcc99645c","metadata":{"jupyter":{"outputs_hidden":true,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["%pip install semantic-link-labs"]},{"cell_type":"markdown","id":"969a29bf","metadata":{},"source":["### Import the library and set initial parameters"]},{"cell_type":"code","execution_count":null,"id":"29c923f8","metadata":{},"outputs":[],"source":["import sempy_labs as labs\n","from sempy_labs import migration, report, directlake\n","\n","dataset_name = '' #Enter the import/DQ semantic model name\n","workspace_name = None #Enter the workspace of the import/DQ semantic model. It set to none it will use the current workspace.\n","new_dataset_name = '' #Enter the new Direct Lake semantic model name\n","new_dataset_workspace_name = None #Enter the workspace where the Direct Lake model will be created. If set to None it will use the current workspace.\n","lakehouse_name = None #Enter the lakehouse to be used for the Direct Lake model. If set to None it will use the lakehouse attached to the notebook.\n","lakehouse_workspace_name = None #Enter the lakehouse workspace. If set to None it will use the new_dataset_workspace_name."]},{"cell_type":"markdown","id":"5a3fe6e8-b8aa-4447-812b-7931831e07fe","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Create the [Power Query Template](https://learn.microsoft.com/power-query/power-query-template) file\n","\n","This encapsulates all of the semantic model's Power Query logic into a single file."]},{"cell_type":"code","execution_count":null,"id":"cde43b47-4ecc-46ae-9125-9674819c7eab","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["migration.create_pqt_file(dataset = dataset_name, workspace = workspace_name)"]},{"cell_type":"markdown","id":"bf945d07-544c-4934-b7a6-cfdb90ca725e","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Import the Power Query Template to Dataflows Gen2\n","\n","- Open the [OneLake file explorer](https://www.microsoft.com/download/details.aspx?id=105222) and sync your files (right click -> Sync from OneLake)\n","\n","- Navigate to your lakehouse. From this window, create a new Dataflows Gen2 and import the Power Query Template file from OneLake (OneLake -> Workspace -> Lakehouse -> Files...), and publish the Dataflows Gen2.\n","\n","
\n","Important!: Make sure to create the Dataflows Gen2 from within the lakehouse window. That will ensure that all the tables automatically map to that lakehouse as the destination. Otherwise, you will have to manually map each table to its destination individually.\n","
"]},{"cell_type":"markdown","id":"9975db7d","metadata":{},"source":["### Create the Direct Lake model based on the import/DQ semantic model\n","\n","Calculated columns are not migrated to the Direct Lake model as they are not supported in Direct Lake mode."]},{"cell_type":"code","execution_count":null,"id":"0a3616b5-566e-414e-a225-fb850d6418dc","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["import time\n","labs.create_blank_semantic_model(dataset = new_dataset_name, workspace = new_dataset_workspace_name)\n","\n","time.sleep(2)\n","\n","migration.migrate_calc_tables_to_lakehouse(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name,\n"," workspace = workspace_name,\n"," new_dataset_workspace = new_dataset_workspace_name,\n"," lakehouse = lakehouse_name,\n"," lakehouse_workspace = lakehouse_workspace_name)\n","migration.migrate_tables_columns_to_semantic_model(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name,\n"," workspace = workspace_name,\n"," new_dataset_workspace = new_dataset_workspace_name,\n"," lakehouse = lakehouse_name,\n"," lakehouse_workspace = lakehouse_workspace_name)\n","migration.migrate_calc_tables_to_semantic_model(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name,\n"," workspace = workspace_name,\n"," new_dataset_workspace = new_dataset_workspace_name,\n"," lakehouse = lakehouse_name,\n"," lakehouse_workspace = lakehouse_workspace_name)\n","migration.migrate_model_objects_to_semantic_model(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name,\n"," workspace = workspace_name,\n"," new_dataset_workspace = new_dataset_workspace_name)\n","migration.migrate_field_parameters(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name,\n"," workspace = workspace_name,\n"," new_dataset_workspace = new_dataset_workspace_name)\n","time.sleep(2)\n","migration.refresh_semantic_model(dataset = new_dataset_name, workspace = new_dataset_workspace_name)\n","migration.refresh_calc_tables(dataset = new_dataset_name, workspace = new_dataset_workspace_name)\n","migration.refresh_semantic_model(dataset = new_dataset_name, workspace = new_dataset_workspace_name)"]},{"cell_type":"markdown","id":"bb98bb13","metadata":{},"source":["### Show migrated/unmigrated objects"]},{"cell_type":"code","execution_count":null,"id":"5db2f22c","metadata":{},"outputs":[],"source":["migration.migration_validation(\n"," dataset = dataset_name,\n"," new_dataset = new_dataset_name, \n"," workspace = workspace_name, \n"," new_dataset_workspace = new_dataset_workspace_name)"]},{"cell_type":"markdown","id":"fa244e9d-87c2-4a66-a7e0-be539a0ac7de","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Rebind all reports using the old semantic model to the new Direct Lake semantic model"]},{"cell_type":"code","execution_count":null,"id":"d4e867cc","metadata":{},"outputs":[],"source":["report.report_rebind_all(\n"," dataset = dataset_name,\n"," dataset_workspace = workspace_name,\n"," new_dataset = new_dataset_name,\n"," new_dataset_workpace = new_dataset_workspace_name,\n"," report_workspace = workspace_name)"]},{"cell_type":"markdown","id":"3365d20d","metadata":{},"source":["### Rebind reports one-by-one (optional)"]},{"cell_type":"code","execution_count":null,"id":"056b7180-d7ac-492c-87e7-ac7d0e4bb929","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["report_name = '' # Enter report name which you want to rebind to the new Direct Lake model\n","\n","report.report_rebind(\n"," report = report_name,\n"," dataset = new_dataset_name,\n"," report_workspace=workspace_name,\n"," dataset_workspace = new_dataset_workspace_name)"]},{"cell_type":"markdown","id":"526f2327","metadata":{},"source":["### Show unsupported objects"]},{"cell_type":"code","execution_count":null,"id":"a47376d7","metadata":{},"outputs":[],"source":["dfT, dfC, dfR = directlake.show_unsupported_direct_lake_objects(dataset = dataset_name, workspace = workspace_name)\n","\n","print('Calculated Tables are not supported...')\n","display(dfT)\n","print(\"Learn more about Direct Lake limitations here: https://learn.microsoft.com/power-bi/enterprise/directlake-overview#known-issues-and-limitations\")\n","print('Calculated columns are not supported. Columns of binary data type are not supported.')\n","display(dfC)\n","print('Columns used for relationship cannot be of data type datetime and they also must be of the same data type.')\n","display(dfR)"]},{"cell_type":"markdown","id":"ed08ba4c","metadata":{},"source":["### Schema check between semantic model tables/columns and lakehouse tables/columns\n","\n","This will list any tables/columns which are in the new semantic model but do not exist in the lakehouse"]},{"cell_type":"code","execution_count":null,"id":"03889ba4","metadata":{},"outputs":[],"source":["directlake.direct_lake_schema_compare(dataset = new_dataset_name, workspace = new_dataset_workspace_name)"]},{"cell_type":"markdown","id":"2229963b","metadata":{},"source":["### Show calculated tables which have been migrated to the Direct Lake semantic model as regular tables"]},{"cell_type":"code","execution_count":null,"id":"dd537d90","metadata":{},"outputs":[],"source":["directlake.list_direct_lake_model_calc_tables(dataset = new_dataset_name, workspace = new_dataset_workspace_name)"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.3"},"microsoft":{"language":"python"},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default"},"synapse_widget":{"state":{},"version":"0.1"},"widgets":{}},"nbformat":4,"nbformat_minor":5} diff --git a/notebooks/Model Optimization.ipynb b/notebooks/Model Optimization.ipynb index 0a5a448f..bd000383 100644 --- a/notebooks/Model Optimization.ipynb +++ b/notebooks/Model Optimization.ipynb @@ -1 +1 @@ -{"cells":[{"cell_type":"markdown","id":"5c27dfd1-4fe0-4a97-92e6-ddf78889aa93","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Install the latest .whl package\n","\n","Check [here](https://github.com/microsoft/semantic-link-labs) to see the latest version.\n","\n","Check [here](https://github.com/microsoft/semantic-link-labs/releases) for the library's release notes."]},{"cell_type":"code","execution_count":null,"id":"d5cae9db-cef9-48a8-a351-9c5fcc99645c","metadata":{"jupyter":{"outputs_hidden":true,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["%pip install \"https://raw.githubusercontent.com/microsoft/semantic-link-labs/main/semantic-link-labs-0.4.1-py3-none-any.whl\""]},{"cell_type":"markdown","id":"cd8de5a0","metadata":{},"source":["### Import the library"]},{"cell_type":"code","execution_count":null,"id":"5cc6eedf","metadata":{},"outputs":[],"source":["import sempy_labs as labs\n","from sempy_labs._tom import connect_semantic_model\n","from sempy_labs import lakehouse as lake\n","from sempy_labs import directlake"]},{"cell_type":"markdown","id":"5a3fe6e8-b8aa-4447-812b-7931831e07fe","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Vertipaq Analyzer"]},{"cell_type":"code","execution_count":null,"id":"cde43b47-4ecc-46ae-9125-9674819c7eab","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["labs.vertipaq_analyzer(dataset = '', workspace = None)"]},{"cell_type":"markdown","id":"419a348f","metadata":{},"source":["Export the Vertipaq Analyzer results to a .zip file in your lakehouse"]},{"cell_type":"code","execution_count":null,"id":"8aa239b3","metadata":{},"outputs":[],"source":["labs.vertipaq_analyzer(dataset = '', workspace = None, export = 'zip')"]},{"cell_type":"markdown","id":"2dce0f4f","metadata":{},"source":["Export the Vertipaq Analyzer results to append to delta tables in your lakehouse."]},{"cell_type":"code","execution_count":null,"id":"aef93fc8","metadata":{},"outputs":[],"source":["labs.vertipaq_analyzer(dataset = '', workspace = None, export = 'table')"]},{"cell_type":"markdown","id":"1c62a802","metadata":{},"source":["Visualize the contents of an exported Vertipaq Analzyer .zip file."]},{"cell_type":"code","execution_count":null,"id":"9e349954","metadata":{},"outputs":[],"source":["labs.import_vertipaq_analyzer(folder_path = '', file_name = '')"]},{"cell_type":"markdown","id":"456ce0ff","metadata":{},"source":["### Best Practice Analzyer\n","\n","This runs the [standard rules](https://github.com/microsoft/Analysis-Services/tree/master/BestPracticeRules) for semantic models posted on Microsoft's GitHub."]},{"cell_type":"code","execution_count":null,"id":"0a3616b5-566e-414e-a225-fb850d6418dc","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["labs.run_model_bpa(dataset = '', workspace = None)"]},{"cell_type":"markdown","id":"6fb32a58","metadata":{},"source":["This runs the rules and exports the results to a table in your lakehouse."]},{"cell_type":"code","execution_count":null,"id":"677851c3","metadata":{},"outputs":[],"source":["labs.run_model_bpa(dataset = '', workspace = None, export = True)"]},{"cell_type":"markdown","id":"8126a1a1","metadata":{},"source":["### Direct Lake\n","\n","Check if any lakehouse tables will hit the [Direct Lake guardrails](https://learn.microsoft.com/power-bi/enterprise/directlake-overview#fallback)."]},{"cell_type":"code","execution_count":null,"id":"e7397b15","metadata":{},"outputs":[],"source":["lake.get_lakehouse_tables(lakehouse = None, workspace = None, extended = True, count_rows = False)"]},{"cell_type":"code","execution_count":null,"id":"b30074cf","metadata":{},"outputs":[],"source":["lake.get_lakehouse_tables(lakehouse = None, workspace = None, extended = True, count_rows = False, export = True)"]},{"cell_type":"markdown","id":"99b84f2b","metadata":{},"source":["Check if any tables in a Direct Lake semantic model will fall back to DirectQuery."]},{"cell_type":"code","execution_count":null,"id":"f837be58","metadata":{},"outputs":[],"source":["directlake.check_fallback_reason(dataset = '', workspace = None)"]},{"cell_type":"markdown","id":"8f6df93e","metadata":{},"source":["### [OPTIMIZE](https://docs.delta.io/latest/optimizations-oss.html) your lakehouse delta tables."]},{"cell_type":"code","execution_count":null,"id":"e0262c9e","metadata":{},"outputs":[],"source":["lake.optimize_lakehouse_tables(tables = ['', ''], lakehouse = None, workspace = None)"]},{"cell_type":"markdown","id":"0091d6a0","metadata":{},"source":["Refresh/reframe your Direct Lake semantic model and restore the columns which were in memory prior to the refresh."]},{"cell_type":"code","execution_count":null,"id":"77eef082","metadata":{},"outputs":[],"source":["directlake.warm_direct_lake_cache_isresident(dataset = '', workspace = None)"]},{"cell_type":"markdown","id":"dae1a210","metadata":{},"source":["Ensure a warm cache for your users by putting the columns of a Direct Lake semantic model into memory based on the contents of a [perspective](https://learn.microsoft.com/analysis-services/tabular-models/perspectives-ssas-tabular?view=asallproducts-allversions).\n","\n","Perspectives can be created either in [Tabular Editor 3](https://github.com/TabularEditor/TabularEditor3/releases/latest) or in [Tabular Editor 2](https://github.com/TabularEditor/TabularEditor/releases/latest) using the [Perspective Editor](https://www.elegantbi.com/post/perspectiveeditor)."]},{"cell_type":"code","execution_count":null,"id":"43297001","metadata":{},"outputs":[],"source":["directlake.warm_direct_lake_cache_perspective(dataset = '', workspace = None, perspective = '', add_dependencies = True)"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","language":"Python","name":"synapse_pyspark"},"language_info":{"name":"python"},"microsoft":{"language":"python"},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default"},"synapse_widget":{"state":{},"version":"0.1"},"widgets":{}},"nbformat":4,"nbformat_minor":5} +{"cells":[{"cell_type":"markdown","id":"5c27dfd1-4fe0-4a97-92e6-ddf78889aa93","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Install the latest .whl package\n","\n","Check [here](https://pypi.org/project/semantic-link-labs/) to see the latest version."]},{"cell_type":"code","execution_count":null,"id":"d5cae9db-cef9-48a8-a351-9c5fcc99645c","metadata":{"jupyter":{"outputs_hidden":true,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["%pip install semantic-link-labs"]},{"cell_type":"markdown","id":"cd8de5a0","metadata":{},"source":["### Import the library"]},{"cell_type":"code","execution_count":null,"id":"5cc6eedf","metadata":{},"outputs":[],"source":["import sempy_labs as labs\n","from sempy_labs import lakehouse as lake\n","from sempy_labs import directlake"]},{"cell_type":"markdown","id":"5a3fe6e8-b8aa-4447-812b-7931831e07fe","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Vertipaq Analyzer"]},{"cell_type":"code","execution_count":null,"id":"cde43b47-4ecc-46ae-9125-9674819c7eab","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["labs.vertipaq_analyzer(dataset = '', workspace = None)"]},{"cell_type":"markdown","id":"419a348f","metadata":{},"source":["Export the Vertipaq Analyzer results to a .zip file in your lakehouse"]},{"cell_type":"code","execution_count":null,"id":"8aa239b3","metadata":{},"outputs":[],"source":["labs.vertipaq_analyzer(dataset = '', workspace = None, export = 'zip')"]},{"cell_type":"markdown","id":"2dce0f4f","metadata":{},"source":["Export the Vertipaq Analyzer results to append to delta tables in your lakehouse."]},{"cell_type":"code","execution_count":null,"id":"aef93fc8","metadata":{},"outputs":[],"source":["labs.vertipaq_analyzer(dataset = '', workspace = None, export = 'table')"]},{"cell_type":"markdown","id":"1c62a802","metadata":{},"source":["Visualize the contents of an exported Vertipaq Analzyer .zip file."]},{"cell_type":"code","execution_count":null,"id":"9e349954","metadata":{},"outputs":[],"source":["labs.import_vertipaq_analyzer(folder_path = '', file_name = '')"]},{"cell_type":"markdown","id":"456ce0ff","metadata":{},"source":["### Best Practice Analzyer\n","\n","This runs the [standard rules](https://github.com/microsoft/Analysis-Services/tree/master/BestPracticeRules) for semantic models posted on Microsoft's GitHub."]},{"cell_type":"code","execution_count":null,"id":"0a3616b5-566e-414e-a225-fb850d6418dc","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["labs.run_model_bpa(dataset = '', workspace = None)"]},{"cell_type":"markdown","id":"6fb32a58","metadata":{},"source":["This runs the rules and exports the results to a table in your lakehouse."]},{"cell_type":"code","execution_count":null,"id":"677851c3","metadata":{},"outputs":[],"source":["labs.run_model_bpa(dataset = '', workspace = None, export = True)"]},{"cell_type":"markdown","id":"8126a1a1","metadata":{},"source":["### Direct Lake\n","\n","Check if any lakehouse tables will hit the [Direct Lake guardrails](https://learn.microsoft.com/power-bi/enterprise/directlake-overview#fallback)."]},{"cell_type":"code","execution_count":null,"id":"e7397b15","metadata":{},"outputs":[],"source":["lake.get_lakehouse_tables(lakehouse = None, workspace = None, extended = True, count_rows = False)"]},{"cell_type":"code","execution_count":null,"id":"b30074cf","metadata":{},"outputs":[],"source":["lake.get_lakehouse_tables(lakehouse = None, workspace = None, extended = True, count_rows = False, export = True)"]},{"cell_type":"markdown","id":"99b84f2b","metadata":{},"source":["Check if any tables in a Direct Lake semantic model will fall back to DirectQuery."]},{"cell_type":"code","execution_count":null,"id":"f837be58","metadata":{},"outputs":[],"source":["directlake.check_fallback_reason(dataset = '', workspace = None)"]},{"cell_type":"markdown","id":"8f6df93e","metadata":{},"source":["### [OPTIMIZE](https://docs.delta.io/latest/optimizations-oss.html) your lakehouse delta tables."]},{"cell_type":"code","execution_count":null,"id":"e0262c9e","metadata":{},"outputs":[],"source":["lake.optimize_lakehouse_tables(tables = ['', ''], lakehouse = None, workspace = None)"]},{"cell_type":"markdown","id":"0091d6a0","metadata":{},"source":["Refresh/reframe your Direct Lake semantic model and restore the columns which were in memory prior to the refresh."]},{"cell_type":"code","execution_count":null,"id":"77eef082","metadata":{},"outputs":[],"source":["directlake.warm_direct_lake_cache_isresident(dataset = '', workspace = None)"]},{"cell_type":"markdown","id":"dae1a210","metadata":{},"source":["Ensure a warm cache for your users by putting the columns of a Direct Lake semantic model into memory based on the contents of a [perspective](https://learn.microsoft.com/analysis-services/tabular-models/perspectives-ssas-tabular?view=asallproducts-allversions).\n","\n","Perspectives can be created either in [Tabular Editor 3](https://github.com/TabularEditor/TabularEditor3/releases/latest) or in [Tabular Editor 2](https://github.com/TabularEditor/TabularEditor/releases/latest) using the [Perspective Editor](https://www.elegantbi.com/post/perspectiveeditor)."]},{"cell_type":"code","execution_count":null,"id":"43297001","metadata":{},"outputs":[],"source":["directlake.warm_direct_lake_cache_perspective(dataset = '', workspace = None, perspective = '', add_dependencies = True)"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","language":"Python","name":"synapse_pyspark"},"language_info":{"name":"python"},"microsoft":{"language":"python"},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default"},"synapse_widget":{"state":{},"version":"0.1"},"widgets":{}},"nbformat":4,"nbformat_minor":5} diff --git a/notebooks/Query Scale Out.ipynb b/notebooks/Query Scale Out.ipynb index a29de474..e2690bd9 100644 --- a/notebooks/Query Scale Out.ipynb +++ b/notebooks/Query Scale Out.ipynb @@ -1 +1 @@ -{"cells":[{"cell_type":"markdown","id":"5c27dfd1-4fe0-4a97-92e6-ddf78889aa93","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Install the latest .whl package\n","\n","Check [here](https://github.com/microsoft/semantic-link-labs) to see the latest version.\n","\n","Check [here](https://github.com/microsoft/semantic-link-labs/releases) for the library's release notes."]},{"cell_type":"code","execution_count":null,"id":"d5cae9db-cef9-48a8-a351-9c5fcc99645c","metadata":{"jupyter":{"outputs_hidden":true,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["%pip install \"https://raw.githubusercontent.com/microsoft/semantic-link-labs/main/semantic-link-labs-0.4.1-py3-none-any.whl\""]},{"cell_type":"markdown","id":"b195eae8","metadata":{},"source":["### Import the library and set the initial parameters"]},{"cell_type":"code","execution_count":null,"id":"1344e286","metadata":{},"outputs":[],"source":["import sempy_labs as labs\n","dataset = '' # Enter your dataset name\n","workspace = None # Enter your workspace name (if set to None it will use the workspace in which the notebook is running)"]},{"cell_type":"markdown","id":"5a3fe6e8-b8aa-4447-812b-7931831e07fe","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### View [Query Scale Out](https://learn.microsoft.com/power-bi/enterprise/service-premium-scale-out) (QSO) settings"]},{"cell_type":"code","execution_count":null,"id":"9e349954","metadata":{},"outputs":[],"source":["labs.list_qso_settings(dataset = dataset, workspace = workspace )"]},{"cell_type":"markdown","id":"b0717cbb","metadata":{},"source":["### [Configure Query Scale Out](https://learn.microsoft.com/power-bi/enterprise/service-premium-scale-out-configure)\n","Setting 'auto_sync' to True will ensure that the semantic model automatically syncs read-only replicas. Setting this to False will necessitate syncing the replicas (i.e. via the qso_sync function).\n","\n","The 'max_read_only_replicas' is the maximum number of read-only replicas for the semantic model (0-64, -1 for automatic number of replicas).\n"]},{"cell_type":"code","execution_count":null,"id":"ec37dd14","metadata":{},"outputs":[],"source":["labs.set_qso(dataset = dataset, auto_sync = False, max_read_only_replicas = -1, workspace = workspace)"]},{"cell_type":"markdown","id":"5d6beadd","metadata":{},"source":["### Sync Query Scale Out replicas"]},{"cell_type":"code","execution_count":null,"id":"7ca10963","metadata":{},"outputs":[],"source":["labs.qso_sync(dataset = dataset, workspace = workspace)"]},{"cell_type":"markdown","id":"719f428f","metadata":{},"source":["### Check Query Scale Out Sync Status"]},{"cell_type":"code","execution_count":null,"id":"db6f197c","metadata":{},"outputs":[],"source":["dfA, dfB = labs.qso_sync_status(dataset = dataset, workspace = workspace)\n","display(dfA)\n","display(dfB)"]},{"cell_type":"markdown","id":"e92cdf34","metadata":{},"source":["### Disable Query Scale Out"]},{"cell_type":"code","execution_count":null,"id":"0624d649","metadata":{},"outputs":[],"source":["labs.disable_qso(dataset = dataset, workspace = workspace)"]},{"cell_type":"markdown","id":"786d89bc","metadata":{},"source":["### Enable large semantic model format"]},{"cell_type":"code","execution_count":null,"id":"d521b228","metadata":{},"outputs":[],"source":["labs.set_semantic_model_storage_format(dataset = dataset, storage_format = 'Large', workspace = workspace)"]},{"cell_type":"markdown","id":"e90c20e9","metadata":{},"source":["### Disable large semantic model format"]},{"cell_type":"code","execution_count":null,"id":"433220b2","metadata":{},"outputs":[],"source":["labs.set_semantic_model_storage_format(dataset = dataset, storage_format = 'Small', workspace = workspace)"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","language":"Python","name":"synapse_pyspark"},"language_info":{"name":"python"},"microsoft":{"language":"python"},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default"},"synapse_widget":{"state":{},"version":"0.1"},"widgets":{}},"nbformat":4,"nbformat_minor":5} +{"cells":[{"cell_type":"markdown","id":"5c27dfd1-4fe0-4a97-92e6-ddf78889aa93","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Install the latest .whl package\n","\n","Check [here](https://pypi.org/project/semantic-link-labs/) to see the latest version."]},{"cell_type":"code","execution_count":null,"id":"d5cae9db-cef9-48a8-a351-9c5fcc99645c","metadata":{"jupyter":{"outputs_hidden":true,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["%pip install semantic-link-labs"]},{"cell_type":"markdown","id":"b195eae8","metadata":{},"source":["### Import the library and set the initial parameters"]},{"cell_type":"code","execution_count":null,"id":"1344e286","metadata":{},"outputs":[],"source":["import sempy_labs as labs\n","dataset = '' # Enter your dataset name\n","workspace = None # Enter your workspace name (if set to None it will use the workspace in which the notebook is running)"]},{"cell_type":"markdown","id":"5a3fe6e8-b8aa-4447-812b-7931831e07fe","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### View [Query Scale Out](https://learn.microsoft.com/power-bi/enterprise/service-premium-scale-out) (QSO) settings"]},{"cell_type":"code","execution_count":null,"id":"9e349954","metadata":{},"outputs":[],"source":["labs.list_qso_settings(dataset = dataset, workspace = workspace )"]},{"cell_type":"markdown","id":"b0717cbb","metadata":{},"source":["### [Configure Query Scale Out](https://learn.microsoft.com/power-bi/enterprise/service-premium-scale-out-configure)\n","Setting 'auto_sync' to True will ensure that the semantic model automatically syncs read-only replicas. Setting this to False will necessitate syncing the replicas (i.e. via the qso_sync function).\n","\n","The 'max_read_only_replicas' is the maximum number of read-only replicas for the semantic model (0-64, -1 for automatic number of replicas).\n"]},{"cell_type":"code","execution_count":null,"id":"ec37dd14","metadata":{},"outputs":[],"source":["labs.set_qso(dataset = dataset, auto_sync = False, max_read_only_replicas = -1, workspace = workspace)"]},{"cell_type":"markdown","id":"5d6beadd","metadata":{},"source":["### Sync Query Scale Out replicas"]},{"cell_type":"code","execution_count":null,"id":"7ca10963","metadata":{},"outputs":[],"source":["labs.qso_sync(dataset = dataset, workspace = workspace)"]},{"cell_type":"markdown","id":"719f428f","metadata":{},"source":["### Check Query Scale Out Sync Status"]},{"cell_type":"code","execution_count":null,"id":"db6f197c","metadata":{},"outputs":[],"source":["dfA, dfB = labs.qso_sync_status(dataset = dataset, workspace = workspace)\n","display(dfA)\n","display(dfB)"]},{"cell_type":"markdown","id":"e92cdf34","metadata":{},"source":["### Disable Query Scale Out"]},{"cell_type":"code","execution_count":null,"id":"0624d649","metadata":{},"outputs":[],"source":["labs.disable_qso(dataset = dataset, workspace = workspace)"]},{"cell_type":"markdown","id":"786d89bc","metadata":{},"source":["### Enable large semantic model format"]},{"cell_type":"code","execution_count":null,"id":"d521b228","metadata":{},"outputs":[],"source":["labs.set_semantic_model_storage_format(dataset = dataset, storage_format = 'Large', workspace = workspace)"]},{"cell_type":"markdown","id":"e90c20e9","metadata":{},"source":["### Disable large semantic model format"]},{"cell_type":"code","execution_count":null,"id":"433220b2","metadata":{},"outputs":[],"source":["labs.set_semantic_model_storage_format(dataset = dataset, storage_format = 'Small', workspace = workspace)"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","language":"Python","name":"synapse_pyspark"},"language_info":{"name":"python"},"microsoft":{"language":"python"},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default"},"synapse_widget":{"state":{},"version":"0.1"},"widgets":{}},"nbformat":4,"nbformat_minor":5} diff --git a/notebooks/Tabular Object Model.ipynb b/notebooks/Tabular Object Model.ipynb index fe6df427..8bf70b2e 100644 --- a/notebooks/Tabular Object Model.ipynb +++ b/notebooks/Tabular Object Model.ipynb @@ -1 +1 @@ -{"cells":[{"cell_type":"markdown","id":"5c27dfd1-4fe0-4a97-92e6-ddf78889aa93","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Install the latest .whl package\n","\n","Check [here](https://github.com/microsoft/semantic-link-labs) to see the latest version.\n","\n","Check [here](https://github.com/microsoft/semantic-link-labs/releases) for the library's release notes."]},{"cell_type":"code","execution_count":null,"id":"d5cae9db-cef9-48a8-a351-9c5fcc99645c","metadata":{"jupyter":{"outputs_hidden":true,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["%pip install \"https://raw.githubusercontent.com/microsoft/semantic-link-labs/main/semantic-link-labs-0.4.1-py3-none-any.whl\""]},{"cell_type":"markdown","id":"5a3fe6e8-b8aa-4447-812b-7931831e07fe","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Connect to the [Tabular Object Model](https://learn.microsoft.com/analysis-services/tom/introduction-to-the-tabular-object-model-tom-in-analysis-services-amo?view=asallproducts-allversions) ([TOM](https://learn.microsoft.com/dotnet/api/microsoft.analysisservices.tabular.model?view=analysisservices-dotnet))\n","Setting the 'readonly' property to False enables read/write mode. This allows changes to be made to the semantic model."]},{"cell_type":"code","execution_count":null,"id":"cde43b47-4ecc-46ae-9125-9674819c7eab","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["import sempy_labs as labs\n","from sempy_labs._tom import connect_semantic_model\n","\n","dataset = '' # Enter dataset name\n","workspace = None # Enter workspace name\n","\n","with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," print(t.Name)"]},{"cell_type":"markdown","id":"fc6b277e","metadata":{},"source":["### Make changes to a semantic model using custom functions\n","Note that the custom functions have additional optional parameters (which may not be used in the examples below) for adding properties to model objects. Check the [documentation](https://github.com/m-kovalsky/fabric_cat_tools) to see all available parameters for each function."]},{"cell_type":"markdown","id":"402a477c","metadata":{},"source":["#### Add measure(s) to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"bdaaaa5c","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_measure(table_name ='Internet Sales', measure_name = 'Sales Amount', expression = \"SUM('Internet Sales'[SalesAmount])\")\n"," tom.add_measure(table_name ='Internet Sales', measure_name = 'Order Quantity', expression = \"SUM('Internet Sales'[OrderQty])\") "]},{"cell_type":"code","execution_count":null,"id":"a53a544b","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," if t.Name == 'Internet Sales':\n"," tom.add_measure(table_name = t.Name, measure_name = 'Sales Amount', expression = \"SUM('Internet Sales'[SalesAmount])\")\n"," tom.add_measure(table_name = t.Name, measure_name = 'Order Quantity', expression = \"SUM('Internet Sales'[OrderQty])\")"]},{"cell_type":"markdown","id":"1cb1632f","metadata":{},"source":["#### Add column(s) to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"81a22749","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_data_column(table_name ='Product', column_name = 'Size Range', source_column = 'SizeRange', data_type = 'Int64')\n"," tom.add_data_column(table_name = 'Segment', column_name = 'Summary Segment', source_column = 'SummarySegment', data_type = 'String')\n","\n"," tom.add_calculated_column(table_name = 'Internet Sales', column_name = 'GrossMargin', expression = \"'Internet Sales'[SalesAmount] - 'Internet Sales'[ProductCost]\", data_type = 'Decimal')"]},{"cell_type":"code","execution_count":null,"id":"053b6516","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," if t.Name == 'Product':\n"," tom.add_data_column(table_name = t.Name, column_name = 'Size Range', source_column = 'SizeRange', data_type = 'Int64')\n"," elif t.Name == 'Segment':\n"," tom.add_data_column(table_name = t.Name, column_name = 'Summary Segment', source_column = 'SummarySegment', data_type = 'String')\n"," elif t.Name == 'Internet Sales':\n"," tom.add_calculated_column(table_name = t.Name, column_name = 'GrossMargin', expression = \"'Internet Sales'[SalesAmount] - 'Internet Sales'[ProductCost]\", data_type = 'Decimal')"]},{"cell_type":"markdown","id":"f53dcca7","metadata":{},"source":["#### Add hierarchies to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"a9309e23","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_hierarchy(table_name = 'Geography', hierarchy_name = 'Geo Hierarchy', levels = ['Continent', 'Country', 'State', 'City'])"]},{"cell_type":"code","execution_count":null,"id":"a04281ce","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," if t.Name == 'Geography':\n"," tom.add_hierarchy(table_name = t.Name, hierarchy_name = 'Geo Hierarchy', levels = ['Continent', 'Country', 'State', 'City'])"]},{"cell_type":"markdown","id":"47c06a4f","metadata":{},"source":["#### Add relationship(s) to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"e8cd7bbf","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_relationship(\n"," from_table = 'Internet Sales', from_column = 'ProductKey',\n"," to_table = 'Product', to_column = 'ProductKey', \n"," from_cardinality = 'Many', to_cardinality = 'One')"]},{"cell_type":"markdown","id":"3cc7f11e","metadata":{},"source":["#### Add a table with an M partition to a semantic model"]},{"cell_type":"code","execution_count":null,"id":"0f5dd66a","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," table_name = 'Sales'\n"," tom.add_table(name = table_name)\n"," tom.add_m_partition(table_name = table_name, partition_name = table_name, expression = 'let....')"]},{"cell_type":"markdown","id":"ea389123","metadata":{},"source":["#### Add a table with an entity partition to a Direct Lake semantic model "]},{"cell_type":"code","execution_count":null,"id":"f75387d1","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," table_name = 'Sales'\n"," tom.add_table(name = table_name)\n"," tom.add_entity_partition(table_name = table_name, entity_name = table_name)"]},{"cell_type":"markdown","id":"e74d0f54","metadata":{},"source":["#### Add a calculated table (and columns) to a semantic model"]},{"cell_type":"code","execution_count":null,"id":"934f7315","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," table_name = 'Sales'\n"," tom.add_calculated_table(name = table_name, expression = \"DISTINCT('Product'[Color])\")\n"," tom.add_calculated_table_column(table_name = table_name, column_name = 'Color', source_column = \"'Product[Color]\", data_type = 'String')"]},{"cell_type":"markdown","id":"0e7088b7","metadata":{},"source":["#### Add role(s) to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"ad60ebb9","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_role(role_name = 'Reader')"]},{"cell_type":"markdown","id":"c541f81a","metadata":{},"source":["#### Set row level security (RLS) to the semantic model\n","This adds row level security (or updates it if it already exists)"]},{"cell_type":"code","execution_count":null,"id":"98603a08","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.set_rls(role_name ='Reader', table_name = 'Product', filter_expression = \"'Dim Product'[Color] = \\\"Blue\\\"\")"]},{"cell_type":"code","execution_count":null,"id":"effea009","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for r in tom.model.Roles:\n"," if r.Name == 'Reader':\n"," tom.set_rls(role_name = r.Name, table_name = 'Product', filter_expression = \"'Dim Product'[Color] = \\\"Blue\\\"\")"]},{"cell_type":"markdown","id":"7fa7a03c","metadata":{},"source":["#### Set object level security (OLS) to the semantic model\n","This adds row level security (or updates it if it already exists)"]},{"cell_type":"code","execution_count":null,"id":"dd0def9d","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.set_ols(role_name = 'Reader', table_name = 'Product', column_name = 'Size', permission = 'None')"]},{"cell_type":"code","execution_count":null,"id":"7a389dc7","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for r in tom.model.Roles:\n"," if r.Name == 'Reader':\n"," for t in tom.model.Tables:\n"," if t.Name == 'Product':\n"," tom.set_ols(role_name = r.Name, table_name = t.Name, column_name = 'Size', permission = 'None')"]},{"cell_type":"markdown","id":"d0f7ccd1","metadata":{},"source":["#### Add calculation groups and calculation items to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"97f4708b","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_calculation_group(name = 'MyCalcGroup')"]},{"cell_type":"code","execution_count":null,"id":"fef68832","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_calculation_item(table_name = 'MyCalcGroup', calculation_item_name = 'YTD', expression = \"CALCULATE(SELECTEDMEASURE(), DATESYTD('Calendar'[CalendarDate]))\")\n"," tom.add_calculation_item(table_name = 'MyCalcGroup', calculation_item_name = 'MTD', expression = \"CALCULATE(SELECTEDMEASURE(), DATESMTD('Calendar'[CalendarDate]))\")"]},{"cell_type":"code","execution_count":null,"id":"c7653dcc","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," if t.Name == 'MyCalcGroup':\n"," tom.add_calculation_item(table_name = t.Name, calculation_item_name = 'YTD', expression = \"CALCULATE(SELECTEDMEASURE(), DATESYTD('Calendar'[CalendarDate]))\")\n"," tom.add_calculation_item(table_name = t.Name, calculation_item_name = 'MTD', expression = \"CALCULATE(SELECTEDMEASURE(), DATESMTD('Calendar'[CalendarDate]))\")"]},{"cell_type":"markdown","id":"c6450c74","metadata":{},"source":["#### Add translations to a semantic model"]},{"cell_type":"code","execution_count":null,"id":"2b616b90","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_translation(language = 'it-IT')"]},{"cell_type":"code","execution_count":null,"id":"dc24c200","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.set_translation(object = tom.model.Tables['Product'], language = 'it-IT', property = 'Name', value = 'Produtto')"]},{"cell_type":"markdown","id":"3048cc95","metadata":{},"source":["#### Add a [Field Parameter](https://learn.microsoft.com/power-bi/create-reports/power-bi-field-parameters) to a semantic model"]},{"cell_type":"code","execution_count":null,"id":"0a94af94","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_field_parameter(table_name = 'Parameter', objects = \"'Product'[Color], [Sales Amount], 'Geography'[Country]\")"]},{"cell_type":"markdown","id":"95aac09a","metadata":{},"source":["#### Remove an object(s) from a semantic model"]},{"cell_type":"code","execution_count":null,"id":"1e2572a8","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," if t.Name == 'Product':\n"," tom.remove_object(object = t.Columns['Size'])\n"," tom.remove_object(object = t.Hierarchies['Product Hierarchy'])"]},{"cell_type":"code","execution_count":null,"id":"bc453177","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.remove_object(object = tom.model.Tables['Product'].Columns['Size'])\n"," tom.remove_object(object = tom.model.Tables['Product'].Hierarchies['Product Hierarchy'])"]},{"cell_type":"markdown","id":"e0d0cb9e","metadata":{},"source":["### Custom functions to loop through non-top-level objects in a semantic model"]},{"cell_type":"code","execution_count":null,"id":"cbe3b1a3","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for c in tom.all_columns():\n"," print(c.Name)"]},{"cell_type":"code","execution_count":null,"id":"3f643e66","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for m in tom.all_measures():\n"," print(m.Name)"]},{"cell_type":"code","execution_count":null,"id":"ed1cde0f","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for p in tom.all_partitions():\n"," print(p.Name)"]},{"cell_type":"code","execution_count":null,"id":"f48014ae","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for h in tom.all_hierarchies():\n"," print(h.Name)"]},{"cell_type":"code","execution_count":null,"id":"9f5e7b72","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for ci in tom.all_calculation_items():\n"," print(ci.Name)"]},{"cell_type":"code","execution_count":null,"id":"3cd9ebc1","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for l in tom.all_levels():\n"," print(l.Name)"]},{"cell_type":"code","execution_count":null,"id":"12c58bad","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for rls in tom.all_rls():\n"," print(rls.Name)"]},{"cell_type":"markdown","id":"1a294bd2","metadata":{},"source":["### See Vertipaq Analyzer stats"]},{"cell_type":"code","execution_count":null,"id":"469660e9","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.set_vertipaq_annotations()\n","\n"," for t in tom.model.Tables:\n"," rc = tom.row_count(object = t)\n"," print(t.Name + ' : ' + str(rc))\n"," for c in t.Columns:\n"," col_size = tom.total_size(column = c)\n"," print(labs.format_dax_object_name(t.Name, c.Name) + ' : ' + str(col_size))"]},{"cell_type":"markdown","id":"1ab26dfd","metadata":{},"source":["### 'UsedIn' functions"]},{"cell_type":"code","execution_count":null,"id":"412bf287","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for c in tom.all_columns():\n"," full_name = labs.format_dax_object_name(c.Parent.Name, c.Name)\n"," for h in tom.used_in_hierarchies(column = c):\n"," print(full_name + ' : ' + h.Name)"]},{"cell_type":"code","execution_count":null,"id":"76556900","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for c in tom.all_columns():\n"," full_name = labs.format_dax_object_name(c.Parent.Name, c.Name)\n"," for r in tom.used_in_relationships(object = c):\n"," rel_name = labs.create_relationship_name(r.FromTable.Name, r.FromColumn.Name, r.ToTable.Name, r.ToColumn.Name)\n"," print(full_name + ' : ' + rel_name)"]},{"cell_type":"code","execution_count":null,"id":"4d9ec24e","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," for r in tom.used_in_relationships(object = t):\n"," rel_name = labs.create_relationship_name(r.FromTable.Name, r.FromColumn.Name, r.ToTable.Name, r.ToColumn.Name)\n"," print(t.Name + ' : ' + rel_name)"]},{"cell_type":"code","execution_count":null,"id":"82251336","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," dep = labs.get_model_calc_dependencies(dataset = dataset, workspace=workspace)\n"," for o in tom.used_in_rls(object = tom.model.Tables['Product'].Columns['Color'], dependencies=dep):\n"," print(o.Name)"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","language":"Python","name":"synapse_pyspark"},"language_info":{"name":"python"},"microsoft":{"language":"python"},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default"},"synapse_widget":{"state":{},"version":"0.1"},"widgets":{}},"nbformat":4,"nbformat_minor":5} +{"cells":[{"cell_type":"markdown","id":"5c27dfd1-4fe0-4a97-92e6-ddf78889aa93","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Install the latest .whl package\n","\n","Check [here](https://pypi.org/project/semantic-link-labs/) to see the latest version."]},{"cell_type":"code","execution_count":null,"id":"d5cae9db-cef9-48a8-a351-9c5fcc99645c","metadata":{"jupyter":{"outputs_hidden":true,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["%pip install semantic-link-labs"]},{"cell_type":"markdown","id":"5a3fe6e8-b8aa-4447-812b-7931831e07fe","metadata":{"nteract":{"transient":{"deleting":false}}},"source":["### Connect to the [Tabular Object Model](https://learn.microsoft.com/analysis-services/tom/introduction-to-the-tabular-object-model-tom-in-analysis-services-amo?view=asallproducts-allversions) ([TOM](https://learn.microsoft.com/dotnet/api/microsoft.analysisservices.tabular.model?view=analysisservices-dotnet))\n","Setting the 'readonly' property to False enables read/write mode. This allows changes to be made to the semantic model."]},{"cell_type":"code","execution_count":null,"id":"cde43b47-4ecc-46ae-9125-9674819c7eab","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["import sempy_labs as labs\n","from sempy_labs.tom.model import connect_semantic_model\n","\n","dataset = '' # Enter dataset name\n","workspace = None # Enter workspace name\n","\n","with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," print(t.Name)"]},{"cell_type":"markdown","id":"fc6b277e","metadata":{},"source":["### Make changes to a semantic model using custom functions\n","Note that the custom functions have additional optional parameters (which may not be used in the examples below) for adding properties to model objects. Check the [documentation](https://github.com/m-kovalsky/fabric_cat_tools) to see all available parameters for each function."]},{"cell_type":"markdown","id":"402a477c","metadata":{},"source":["#### Add measure(s) to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"bdaaaa5c","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_measure(table_name ='Internet Sales', measure_name = 'Sales Amount', expression = \"SUM('Internet Sales'[SalesAmount])\")\n"," tom.add_measure(table_name ='Internet Sales', measure_name = 'Order Quantity', expression = \"SUM('Internet Sales'[OrderQty])\") "]},{"cell_type":"code","execution_count":null,"id":"a53a544b","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," if t.Name == 'Internet Sales':\n"," tom.add_measure(table_name = t.Name, measure_name = 'Sales Amount', expression = \"SUM('Internet Sales'[SalesAmount])\")\n"," tom.add_measure(table_name = t.Name, measure_name = 'Order Quantity', expression = \"SUM('Internet Sales'[OrderQty])\")"]},{"cell_type":"markdown","id":"1cb1632f","metadata":{},"source":["#### Add column(s) to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"81a22749","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_data_column(table_name ='Product', column_name = 'Size Range', source_column = 'SizeRange', data_type = 'Int64')\n"," tom.add_data_column(table_name = 'Segment', column_name = 'Summary Segment', source_column = 'SummarySegment', data_type = 'String')\n","\n"," tom.add_calculated_column(table_name = 'Internet Sales', column_name = 'GrossMargin', expression = \"'Internet Sales'[SalesAmount] - 'Internet Sales'[ProductCost]\", data_type = 'Decimal')"]},{"cell_type":"code","execution_count":null,"id":"053b6516","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," if t.Name == 'Product':\n"," tom.add_data_column(table_name = t.Name, column_name = 'Size Range', source_column = 'SizeRange', data_type = 'Int64')\n"," elif t.Name == 'Segment':\n"," tom.add_data_column(table_name = t.Name, column_name = 'Summary Segment', source_column = 'SummarySegment', data_type = 'String')\n"," elif t.Name == 'Internet Sales':\n"," tom.add_calculated_column(table_name = t.Name, column_name = 'GrossMargin', expression = \"'Internet Sales'[SalesAmount] - 'Internet Sales'[ProductCost]\", data_type = 'Decimal')"]},{"cell_type":"markdown","id":"f53dcca7","metadata":{},"source":["#### Add hierarchies to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"a9309e23","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_hierarchy(table_name = 'Geography', hierarchy_name = 'Geo Hierarchy', levels = ['Continent', 'Country', 'State', 'City'])"]},{"cell_type":"code","execution_count":null,"id":"a04281ce","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," if t.Name == 'Geography':\n"," tom.add_hierarchy(table_name = t.Name, hierarchy_name = 'Geo Hierarchy', levels = ['Continent', 'Country', 'State', 'City'])"]},{"cell_type":"markdown","id":"47c06a4f","metadata":{},"source":["#### Add relationship(s) to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"e8cd7bbf","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_relationship(\n"," from_table = 'Internet Sales', from_column = 'ProductKey',\n"," to_table = 'Product', to_column = 'ProductKey', \n"," from_cardinality = 'Many', to_cardinality = 'One')"]},{"cell_type":"markdown","id":"3cc7f11e","metadata":{},"source":["#### Add a table with an M partition to a semantic model"]},{"cell_type":"code","execution_count":null,"id":"0f5dd66a","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," table_name = 'Sales'\n"," tom.add_table(name = table_name)\n"," tom.add_m_partition(table_name = table_name, partition_name = table_name, expression = 'let....')"]},{"cell_type":"markdown","id":"ea389123","metadata":{},"source":["#### Add a table with an entity partition to a Direct Lake semantic model "]},{"cell_type":"code","execution_count":null,"id":"f75387d1","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," table_name = 'Sales'\n"," tom.add_table(name = table_name)\n"," tom.add_entity_partition(table_name = table_name, entity_name = table_name)"]},{"cell_type":"markdown","id":"e74d0f54","metadata":{},"source":["#### Add a calculated table (and columns) to a semantic model"]},{"cell_type":"code","execution_count":null,"id":"934f7315","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," table_name = 'Sales'\n"," tom.add_calculated_table(name = table_name, expression = \"DISTINCT('Product'[Color])\")\n"," tom.add_calculated_table_column(table_name = table_name, column_name = 'Color', source_column = \"'Product[Color]\", data_type = 'String')"]},{"cell_type":"markdown","id":"0e7088b7","metadata":{},"source":["#### Add role(s) to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"ad60ebb9","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_role(role_name = 'Reader')"]},{"cell_type":"markdown","id":"c541f81a","metadata":{},"source":["#### Set row level security (RLS) to the semantic model\n","This adds row level security (or updates it if it already exists)"]},{"cell_type":"code","execution_count":null,"id":"98603a08","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.set_rls(role_name ='Reader', table_name = 'Product', filter_expression = \"'Dim Product'[Color] = \\\"Blue\\\"\")"]},{"cell_type":"code","execution_count":null,"id":"effea009","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for r in tom.model.Roles:\n"," if r.Name == 'Reader':\n"," tom.set_rls(role_name = r.Name, table_name = 'Product', filter_expression = \"'Dim Product'[Color] = \\\"Blue\\\"\")"]},{"cell_type":"markdown","id":"7fa7a03c","metadata":{},"source":["#### Set object level security (OLS) to the semantic model\n","This adds row level security (or updates it if it already exists)"]},{"cell_type":"code","execution_count":null,"id":"dd0def9d","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.set_ols(role_name = 'Reader', table_name = 'Product', column_name = 'Size', permission = 'None')"]},{"cell_type":"code","execution_count":null,"id":"7a389dc7","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for r in tom.model.Roles:\n"," if r.Name == 'Reader':\n"," for t in tom.model.Tables:\n"," if t.Name == 'Product':\n"," tom.set_ols(role_name = r.Name, table_name = t.Name, column_name = 'Size', permission = 'None')"]},{"cell_type":"markdown","id":"d0f7ccd1","metadata":{},"source":["#### Add calculation groups and calculation items to the semantic model"]},{"cell_type":"code","execution_count":null,"id":"97f4708b","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_calculation_group(name = 'MyCalcGroup')"]},{"cell_type":"code","execution_count":null,"id":"fef68832","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_calculation_item(table_name = 'MyCalcGroup', calculation_item_name = 'YTD', expression = \"CALCULATE(SELECTEDMEASURE(), DATESYTD('Calendar'[CalendarDate]))\")\n"," tom.add_calculation_item(table_name = 'MyCalcGroup', calculation_item_name = 'MTD', expression = \"CALCULATE(SELECTEDMEASURE(), DATESMTD('Calendar'[CalendarDate]))\")"]},{"cell_type":"code","execution_count":null,"id":"c7653dcc","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," if t.Name == 'MyCalcGroup':\n"," tom.add_calculation_item(table_name = t.Name, calculation_item_name = 'YTD', expression = \"CALCULATE(SELECTEDMEASURE(), DATESYTD('Calendar'[CalendarDate]))\")\n"," tom.add_calculation_item(table_name = t.Name, calculation_item_name = 'MTD', expression = \"CALCULATE(SELECTEDMEASURE(), DATESMTD('Calendar'[CalendarDate]))\")"]},{"cell_type":"markdown","id":"c6450c74","metadata":{},"source":["#### Add translations to a semantic model"]},{"cell_type":"code","execution_count":null,"id":"2b616b90","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_translation(language = 'it-IT')"]},{"cell_type":"code","execution_count":null,"id":"dc24c200","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.set_translation(object = tom.model.Tables['Product'], language = 'it-IT', property = 'Name', value = 'Produtto')"]},{"cell_type":"markdown","id":"3048cc95","metadata":{},"source":["#### Add a [Field Parameter](https://learn.microsoft.com/power-bi/create-reports/power-bi-field-parameters) to a semantic model"]},{"cell_type":"code","execution_count":null,"id":"0a94af94","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.add_field_parameter(table_name = 'Parameter', objects = \"'Product'[Color], [Sales Amount], 'Geography'[Country]\")"]},{"cell_type":"markdown","id":"95aac09a","metadata":{},"source":["#### Remove an object(s) from a semantic model"]},{"cell_type":"code","execution_count":null,"id":"1e2572a8","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," if t.Name == 'Product':\n"," tom.remove_object(object = t.Columns['Size'])\n"," tom.remove_object(object = t.Hierarchies['Product Hierarchy'])"]},{"cell_type":"code","execution_count":null,"id":"bc453177","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.remove_object(object = tom.model.Tables['Product'].Columns['Size'])\n"," tom.remove_object(object = tom.model.Tables['Product'].Hierarchies['Product Hierarchy'])"]},{"cell_type":"markdown","id":"e0d0cb9e","metadata":{},"source":["### Custom functions to loop through non-top-level objects in a semantic model"]},{"cell_type":"code","execution_count":null,"id":"cbe3b1a3","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for c in tom.all_columns():\n"," print(c.Name)"]},{"cell_type":"code","execution_count":null,"id":"3f643e66","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for m in tom.all_measures():\n"," print(m.Name)"]},{"cell_type":"code","execution_count":null,"id":"ed1cde0f","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for p in tom.all_partitions():\n"," print(p.Name)"]},{"cell_type":"code","execution_count":null,"id":"f48014ae","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for h in tom.all_hierarchies():\n"," print(h.Name)"]},{"cell_type":"code","execution_count":null,"id":"9f5e7b72","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for ci in tom.all_calculation_items():\n"," print(ci.Name)"]},{"cell_type":"code","execution_count":null,"id":"3cd9ebc1","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for l in tom.all_levels():\n"," print(l.Name)"]},{"cell_type":"code","execution_count":null,"id":"12c58bad","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," for rls in tom.all_rls():\n"," print(rls.Name)"]},{"cell_type":"markdown","id":"1a294bd2","metadata":{},"source":["### See Vertipaq Analyzer stats"]},{"cell_type":"code","execution_count":null,"id":"469660e9","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=False, workspace=workspace) as tom:\n"," tom.set_vertipaq_annotations()\n","\n"," for t in tom.model.Tables:\n"," rc = tom.row_count(object = t)\n"," print(t.Name + ' : ' + str(rc))\n"," for c in t.Columns:\n"," col_size = tom.total_size(column = c)\n"," print(labs.format_dax_object_name(t.Name, c.Name) + ' : ' + str(col_size))"]},{"cell_type":"markdown","id":"1ab26dfd","metadata":{},"source":["### 'UsedIn' functions"]},{"cell_type":"code","execution_count":null,"id":"412bf287","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for c in tom.all_columns():\n"," full_name = labs.format_dax_object_name(c.Parent.Name, c.Name)\n"," for h in tom.used_in_hierarchies(column = c):\n"," print(full_name + ' : ' + h.Name)"]},{"cell_type":"code","execution_count":null,"id":"76556900","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for c in tom.all_columns():\n"," full_name = labs.format_dax_object_name(c.Parent.Name, c.Name)\n"," for r in tom.used_in_relationships(object = c):\n"," rel_name = labs.create_relationship_name(r.FromTable.Name, r.FromColumn.Name, r.ToTable.Name, r.ToColumn.Name)\n"," print(full_name + ' : ' + rel_name)"]},{"cell_type":"code","execution_count":null,"id":"4d9ec24e","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," for t in tom.model.Tables:\n"," for r in tom.used_in_relationships(object = t):\n"," rel_name = labs.create_relationship_name(r.FromTable.Name, r.FromColumn.Name, r.ToTable.Name, r.ToColumn.Name)\n"," print(t.Name + ' : ' + rel_name)"]},{"cell_type":"code","execution_count":null,"id":"82251336","metadata":{},"outputs":[],"source":["with connect_semantic_model(dataset=dataset, readonly=True, workspace=workspace) as tom:\n"," dep = labs.get_model_calc_dependencies(dataset = dataset, workspace=workspace)\n"," for o in tom.used_in_rls(object = tom.model.Tables['Product'].Columns['Color'], dependencies=dep):\n"," print(o.Name)"]}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","language":"Python","name":"synapse_pyspark"},"language_info":{"name":"python"},"microsoft":{"language":"python"},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default"},"synapse_widget":{"state":{},"version":"0.1"},"widgets":{}},"nbformat":4,"nbformat_minor":5} diff --git a/src/sempy_labs/__init__.py b/src/sempy_labs/__init__.py index 349adbf9..3661c1de 100644 --- a/src/sempy_labs/__init__.py +++ b/src/sempy_labs/__init__.py @@ -13,6 +13,8 @@ get_semantic_model_bim, ) from sempy_labs._list_functions import ( + list_semantic_model_objects, + list_shortcuts, get_object_level_security, # list_annotations, # list_columns, @@ -52,7 +54,7 @@ resolve_report_name, # language_validate ) -from sempy_labs._model_auto_build import model_auto_build +#from sempy_labs._model_auto_build import model_auto_build from sempy_labs._model_bpa import model_bpa_rules, run_model_bpa from sempy_labs._model_dependencies import ( measure_dependency_tree, @@ -62,16 +64,15 @@ from sempy_labs._one_lake_integration import ( export_model_to_onelake, ) - -# from sempy_labs._query_scale_out import ( -# qso_sync, -# qso_sync_status, -# set_qso, -# list_qso_settings, -# disable_qso, -# set_semantic_model_storage_format, -# set_workspace_default_storage_format, -# ) +from sempy_labs._query_scale_out import ( + qso_sync, + qso_sync_status, + set_qso, + list_qso_settings, + disable_qso, + set_semantic_model_storage_format, + set_workspace_default_storage_format, +) from sempy_labs._refresh_semantic_model import ( refresh_semantic_model, cancel_dataset_refresh, @@ -82,7 +83,6 @@ # visualize_vertipaq, import_vertipaq_analyzer, ) -from sempy_labs._tom import TOMWrapper, connect_semantic_model __all__ = [ "clear_cache", @@ -129,26 +129,26 @@ "resolve_report_id", "resolve_report_name", #'language_validate', - "model_auto_build", + #"model_auto_build", "model_bpa_rules", "run_model_bpa", "measure_dependency_tree", "get_measure_dependencies", "get_model_calc_dependencies", "export_model_to_onelake", - #'qso_sync', - #'qso_sync_status', - #'set_qso', - #'list_qso_settings', - #'disable_qso', - #'set_semantic_model_storage_format', - #'set_workspace_default_storage_format', + 'qso_sync', + 'qso_sync_status', + 'set_qso', + 'list_qso_settings', + 'disable_qso', + 'set_semantic_model_storage_format', + 'set_workspace_default_storage_format', "refresh_semantic_model", "cancel_dataset_refresh", "translate_semantic_model", "vertipaq_analyzer", #'visualize_vertipaq', "import_vertipaq_analyzer", - "TOMWrapper", - "connect_semantic_model", + "list_semantic_model_objects", + "list_shortcuts" ] diff --git a/src/sempy_labs/_generate_semantic_model.py b/src/sempy_labs/_generate_semantic_model.py index c7b79d00..7f33bce6 100644 --- a/src/sempy_labs/_generate_semantic_model.py +++ b/src/sempy_labs/_generate_semantic_model.py @@ -23,9 +23,8 @@ def create_blank_semantic_model( ---------- dataset : str Name of the semantic model. - compatibility_level : int + compatibility_level : int, default=1605 The compatibility level of the semantic model. - Defaults to 1605. workspace : str, default=None The Fabric workspace name. Defaults to None which resolves to the workspace of the attached lakehouse diff --git a/src/sempy_labs/_helper_functions.py b/src/sempy_labs/_helper_functions.py index e1c13edb..70eda84d 100644 --- a/src/sempy_labs/_helper_functions.py +++ b/src/sempy_labs/_helper_functions.py @@ -1,3 +1,4 @@ +import sempy import sempy.fabric as fabric import re import pandas as pd @@ -199,14 +200,15 @@ def resolve_dataset_name(dataset_id: UUID, workspace: Optional[str] = None): return obj -def resolve_lakehouse_name(lakehouse_id: UUID, workspace: Optional[str] = None): +def resolve_lakehouse_name(lakehouse_id: Optional[UUID] = None, workspace: Optional[str] = None): """ Obtains the name of the Fabric lakehouse. Parameters ---------- - lakehouse_id : UUID + lakehouse_id : UUID, default=None The name of the Fabric lakehouse. + Defaults to None which resolves to the lakehouse attached to the notebook. workspace : str, default=None The Fabric workspace name. Defaults to None which resolves to the workspace of the attached lakehouse @@ -221,6 +223,9 @@ def resolve_lakehouse_name(lakehouse_id: UUID, workspace: Optional[str] = None): if workspace == None: workspace_id = fabric.get_workspace_id() workspace = fabric.resolve_workspace_name(workspace_id) + + if lakehouse_id is None: + lakehouse_id = fabric.get_lakehouse_id() obj = fabric.resolve_item_name( item_id=lakehouse_id, type="Lakehouse", workspace=workspace diff --git a/src/sempy_labs/_list_functions.py b/src/sempy_labs/_list_functions.py index 28de4352..07f0c56f 100644 --- a/src/sempy_labs/_list_functions.py +++ b/src/sempy_labs/_list_functions.py @@ -1,11 +1,15 @@ +import sempy import sempy.fabric as fabric -from sempy_labs._helper_functions import resolve_workspace_name_and_id +from sempy_labs._helper_functions import ( + resolve_workspace_name_and_id, + resolve_lakehouse_name, + create_relationship_name, + resolve_lakehouse_id) import pandas as pd import json, time from pyspark.sql import SparkSession from typing import Optional - def get_object_level_security(dataset: str, workspace: Optional[str] = None): """ Shows the object level security for the semantic model. @@ -1286,7 +1290,7 @@ def list_kpis(dataset: str, workspace: Optional[str] = None): A pandas dataframe showing the KPIs for the semantic model. """ - from ._tom import connect_semantic_model + from .tom.model import connect_semantic_model with connect_semantic_model( dataset=dataset, workspace=workspace, readonly=True @@ -1370,3 +1374,256 @@ def list_workspace_role_assignments(workspace: Optional[str] = None): df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) return df + +def list_semantic_model_objects(dataset: str, workspace: Optional[str] = None): + """ + Shows a list of semantic model objects. + + Parameters + ---------- + dataset : str + Name of the semantic model. + workspace : str, default=None + The Fabric workspace name. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + + Returns + ------- + pandas.DataFrame + A pandas dataframe showing a list of objects in the semantic model + """ + from .tom.model import connect_semantic_model + + df = pd.DataFrame(columns=["Parent Name", "Object Name", "Object Type"]) + with connect_semantic_model( + dataset=dataset, workspace=workspace, readonly=True + ) as tom: + for t in tom._model.Tables: + if t.CalculationGroup is not None: + new_data = { + "Parent Name": t.Parent.Name, + "Object Name": t.Name, + "Object Type": "Calculation Group", + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + for ci in t.CalculationGroup.CalculationItems: + new_data = { + "Parent Name": t.Name, + "Object Name": ci.Name, + "Object Type": str(ci.ObjectType), + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + elif any(str(p.SourceType) == "Calculated" for p in t.Partitions): + new_data = { + "Parent Name": t.Parent.Name, + "Object Name": t.Name, + "Object Type": "Calculated Table", + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + else: + new_data = { + "Parent Name": t.Parent.Name, + "Object Name": t.Name, + "Object Type": str(t.ObjectType), + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + for c in t.Columns: + if str(c.Type) != "RowNumber": + if str(c.Type) == "Calculated": + new_data = { + "Parent Name": c.Parent.Name, + "Object Name": c.Name, + "Object Type": "Calculated Column", + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + else: + new_data = { + "Parent Name": c.Parent.Name, + "Object Name": c.Name, + "Object Type": str(c.ObjectType), + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + for m in t.Measures: + new_data = { + "Parent Name": m.Parent.Name, + "Object Name": m.Name, + "Object Type": str(m.ObjectType), + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + for h in t.Hierarchies: + new_data = { + "Parent Name": h.Parent.Name, + "Object Name": h.Name, + "Object Type": str(h.ObjectType), + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + for l in h.Levels: + new_data = { + "Parent Name": l.Parent.Name, + "Object Name": l.Name, + "Object Type": str(l.ObjectType), + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + for p in t.Partitions: + new_data = { + "Parent Name": p.Parent.Name, + "Object Name": p.Name, + "Object Type": str(p.ObjectType), + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + for r in tom._model.Relationships: + rName = create_relationship_name( + r.FromTable.Name, r.FromColumn.Name, r.ToTable.Name, r.ToColumn.Name + ) + new_data = { + "Parent Name": r.Parent.Name, + "Object Name": rName, + "Object Type": str(r.ObjectType), + } + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + for role in tom._model.Roles: + new_data = { + "Parent Name": role.Parent.Name, + "Object Name": role.Name, + "Object Type": str(role.ObjectType), + } + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + for rls in role.TablePermissions: + new_data = { + "Parent Name": role.Name, + "Object Name": rls.Name, + "Object Type": str(rls.ObjectType), + } + df = pd.concat( + [df, pd.DataFrame(new_data, index=[0])], ignore_index=True + ) + for tr in tom._model.Cultures: + new_data = { + "Parent Name": tr.Parent.Name, + "Object Name": tr.Name, + "Object Type": str(tr.ObjectType), + } + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + for per in tom._model.Perspectives: + new_data = { + "Parent Name": per.Parent.Name, + "Object Name": per.Name, + "Object Type": str(per.ObjectType), + } + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + + return df + +def list_shortcuts( + lakehouse: Optional[str] = None, workspace: Optional[str] = None +) -> pd.DataFrame: + """ + Shows all shortcuts which exist in a Fabric lakehouse. + + Parameters + ---------- + lakehouse : str, default=None + The Fabric lakehouse name. + Defaults to None which resolves to the lakehouse attached to the notebook. + workspace : str, default=None + The name of the Fabric workspace in which lakehouse resides. + Defaults to None which resolves to the workspace of the attached lakehouse + or if no lakehouse attached, resolves to the workspace of the notebook. + + Returns + ------- + pandas.DataFrame + A pandas dataframe showing all the shortcuts which exist in the specified lakehouse. + """ + + (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) + + if lakehouse == None: + lakehouse_id = fabric.get_lakehouse_id() + lakehouse = resolve_lakehouse_name(lakehouse_id, workspace) + else: + lakehouse_id = resolve_lakehouse_id(lakehouse, workspace) + + df = pd.DataFrame( + columns=[ + "Shortcut Name", + "Shortcut Path", + "Source", + "Source Lakehouse Name", + "Source Workspace Name", + "Source Path", + "Source Connection ID", + "Source Location", + "Source SubPath", + ] + ) + + client = fabric.FabricRestClient() + response = client.get( + f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts" + ) + if response.status_code == 200: + for s in response.json()["value"]: + shortcutName = s["name"] + shortcutPath = s["path"] + source = list(s["target"].keys())[0] + ( + sourceLakehouseName, + sourceWorkspaceName, + sourcePath, + connectionId, + location, + subpath, + ) = (None, None, None, None, None, None) + if source == "oneLake": + sourceLakehouseId = s["target"][source]["itemId"] + sourcePath = s["target"][source]["path"] + sourceWorkspaceId = s["target"][source]["workspaceId"] + sourceWorkspaceName = fabric.resolve_workspace_name(sourceWorkspaceId) + sourceLakehouseName = resolve_lakehouse_name( + sourceLakehouseId, sourceWorkspaceName + ) + else: + connectionId = s["target"][source]["connectionId"] + location = s["target"][source]["location"] + subpath = s["target"][source]["subpath"] + + new_data = { + "Shortcut Name": shortcutName, + "Shortcut Path": shortcutPath, + "Source": source, + "Source Lakehouse Name": sourceLakehouseName, + "Source Workspace Name": sourceWorkspaceName, + "Source Path": sourcePath, + "Source Connection ID": connectionId, + "Source Location": location, + "Source SubPath": subpath, + } + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + + print( + f"This function relies on an API which is not yet official as of May 21, 2024. Once the API becomes official this function will work as expected." + ) + return df \ No newline at end of file diff --git a/src/sempy_labs/_model_auto_build.py b/src/sempy_labs/_model_auto_build.py index 0fa7070e..b011e641 100644 --- a/src/sempy_labs/_model_auto_build.py +++ b/src/sempy_labs/_model_auto_build.py @@ -1,7 +1,7 @@ import sempy import sempy.fabric as fabric import pandas as pd -from sempy_labs._tom import connect_semantic_model +from sempy_labs.tom.model import connect_semantic_model from sempy_labs._generate_semantic_model import create_blank_semantic_model from sempy_labs.directlake._get_shared_expression import get_shared_expression from typing import List, Optional, Union diff --git a/src/sempy_labs/_one_lake_integration.py b/src/sempy_labs/_one_lake_integration.py index dc155c47..c335d880 100644 --- a/src/sempy_labs/_one_lake_integration.py +++ b/src/sempy_labs/_one_lake_integration.py @@ -1,3 +1,4 @@ +import sempy import sempy.fabric as fabric import pandas as pd from typing import Optional diff --git a/src/sempy_labs/_translations.py b/src/sempy_labs/_translations.py index c501d39a..b9335f90 100644 --- a/src/sempy_labs/_translations.py +++ b/src/sempy_labs/_translations.py @@ -35,7 +35,7 @@ def translate_semantic_model( from synapse.ml.services import Translate from pyspark.sql.functions import col, flatten from pyspark.sql import SparkSession - from ._tom import connect_semantic_model + from .tom.model import connect_semantic_model if isinstance(languages, str): languages = [languages] diff --git a/src/sempy_labs/directlake/_directlake_schema_sync.py b/src/sempy_labs/directlake/_directlake_schema_sync.py index 788743d2..64edae86 100644 --- a/src/sempy_labs/directlake/_directlake_schema_sync.py +++ b/src/sempy_labs/directlake/_directlake_schema_sync.py @@ -2,6 +2,7 @@ import sempy.fabric as fabric import pandas as pd from sempy_labs.lakehouse._get_lakehouse_columns import get_lakehouse_columns +from sempy_labs.tom.model import connect_semantic_model from sempy_labs._helper_functions import ( format_dax_object_name, resolve_lakehouse_name, @@ -46,8 +47,7 @@ def direct_lake_schema_sync( import System if workspace == None: - workspace_id = fabric.get_workspace_id() - workspace = fabric.resolve_workspace_name(workspace_id) + workspace = fabric.resolve_workspace_name() if lakehouse_workspace is None: lakehouse_workspace = workspace @@ -93,36 +93,38 @@ def direct_lake_schema_sync( "double": "Double", } - tom_server = fabric.create_tom_server(readonly=False, workspace=workspace) - m = tom_server.Databases.GetByName(dataset).Model - for i, r in lc_filt.iterrows(): - lakeTName = r["Table Name"] - lakeCName = r["Column Name"] - fullColName = r["Full Column Name"] - dType = r["Data Type"] - - if fullColName not in dfC_filt["Column Object"].values: - dfL = dfP_filt[dfP_filt["Query"] == lakeTName] - tName = dfL["Table Name"].iloc[0] - if add_to_model: - col = TOM.DataColumn() - col.Name = lakeCName - col.SourceColumn = lakeCName - dt = mapping.get(dType) - try: - col.DataType = System.Enum.Parse(TOM.DataType, dt) - except: + with connect_semantic_model( + dataset=dataset, readonly=False, workspace=workspace + ) as tom: + + for i, r in lc_filt.iterrows(): + lakeTName = r["Table Name"] + lakeCName = r["Column Name"] + fullColName = r["Full Column Name"] + dType = r["Data Type"] + + if fullColName not in dfC_filt["Column Object"].values: + dfL = dfP_filt[dfP_filt["Query"] == lakeTName] + tName = dfL["Table Name"].iloc[0] + if add_to_model: + col = TOM.DataColumn() + col.Name = lakeCName + col.SourceColumn = lakeCName + dt = mapping.get(dType) + try: + col.DataType = System.Enum.Parse(TOM.DataType, dt) + except: + print( + f"{icons.red_dot} '{dType}' data type is not mapped properly to the semantic model data types." + ) + return + + tom.model.Tables[tName].Columns.Add(col) + print( + f"{icons.green_dot} The '{lakeCName}' column has been added to the '{tName}' table as a '{dt}' data type within the '{dataset}' semantic model within the '{workspace}' workspace." + ) + else: print( - f"{icons.red_dot} '{dType}' data type is not mapped properly to the semantic model data types." + f"{icons.yellow_dot} The {fullColName} column exists in the lakehouse but not in the '{tName}' table in the '{dataset}' semantic model within the '{workspace}' workspace." ) - return - - m.Tables[tName].Columns.Add(col) - print( - f"{icons.green_dot} The '{lakeCName}' column has been added to the '{tName}' table as a '{dt}' data type within the '{dataset}' semantic model within the '{workspace}' workspace." - ) - else: - print( - f"{icons.yellow_dot} The {fullColName} column exists in the lakehouse but not in the '{tName}' table in the '{dataset}' semantic model within the '{workspace}' workspace." - ) - m.SaveChanges() + diff --git a/src/sempy_labs/directlake/_get_directlake_lakehouse.py b/src/sempy_labs/directlake/_get_directlake_lakehouse.py index 92d4ff7b..63d03191 100644 --- a/src/sempy_labs/directlake/_get_directlake_lakehouse.py +++ b/src/sempy_labs/directlake/_get_directlake_lakehouse.py @@ -7,7 +7,6 @@ ) from typing import Optional, Tuple from uuid import UUID -from sempy_labs._helper_functions import resolve_workspace_name_and_id import sempy_labs._icons as icons def get_direct_lake_lakehouse( diff --git a/src/sempy_labs/directlake/_list_directlake_model_calc_tables.py b/src/sempy_labs/directlake/_list_directlake_model_calc_tables.py index 392e3aca..c8422ca6 100644 --- a/src/sempy_labs/directlake/_list_directlake_model_calc_tables.py +++ b/src/sempy_labs/directlake/_list_directlake_model_calc_tables.py @@ -2,6 +2,7 @@ import sempy.fabric as fabric import pandas as pd from sempy_labs._list_functions import list_tables, list_annotations +from sempy_labs.tom.model import connect_semantic_model from typing import Optional from sempy._utils._log import log import sempy_labs._icons as icons @@ -27,28 +28,30 @@ def list_direct_lake_model_calc_tables(dataset: str, workspace: Optional[str] = """ if workspace == None: - workspace_id = fabric.get_workspace_id() - workspace = fabric.resolve_workspace_name(workspace_id) + workspace = fabric.resolve_workspace_name() df = pd.DataFrame(columns=["Table Name", "Source Expression"]) - dfP = fabric.list_partitions(dataset=dataset, workspace=workspace) - dfP_filt = dfP[dfP["Mode"] == "DirectLake"] - - if len(dfP_filt) == 0: - print(f"{icons.yellow_dot} The '{dataset}' semantic model is not in Direct Lake mode.") - else: - dfA = list_annotations(dataset, workspace) - dfT = list_tables(dataset, workspace) - dfA_filt = dfA[ - (dfA["Object Type"] == "Model") & (dfA["Annotation Name"].isin(dfT["Name"])) - ] - - for i, r in dfA_filt.iterrows(): - tName = r["Annotation Name"] - se = r["Annotation Value"] - - new_data = {"Table Name": tName, "Source Expression": se} - df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) - - return df + with connect_semantic_model( + dataset=dataset, readonly=True, workspace=workspace + ) as tom: + + is_direct_lake = tom.is_direct_lake() + + if not is_direct_lake: + print(f"{icons.yellow_dot} The '{dataset}' semantic model is not in Direct Lake mode.") + else: + dfA = list_annotations(dataset, workspace) + dfT = list_tables(dataset, workspace) + dfA_filt = dfA[ + (dfA["Object Type"] == "Model") & (dfA["Annotation Name"].isin(dfT["Name"])) + ] + + for i, r in dfA_filt.iterrows(): + tName = r["Annotation Name"] + se = r["Annotation Value"] + + new_data = {"Table Name": tName, "Source Expression": se} + df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) + + return df diff --git a/src/sempy_labs/directlake/_show_unsupported_directlake_objects.py b/src/sempy_labs/directlake/_show_unsupported_directlake_objects.py index 5fc88b95..8e625e8a 100644 --- a/src/sempy_labs/directlake/_show_unsupported_directlake_objects.py +++ b/src/sempy_labs/directlake/_show_unsupported_directlake_objects.py @@ -30,8 +30,7 @@ def show_unsupported_direct_lake_objects( pd.options.mode.chained_assignment = None if workspace == None: - workspace_id = fabric.get_workspace_id() - workspace = fabric.resolve_workspace_name(workspace_id) + workspace = fabric.resolve_workspace_name() dfT = list_tables(dataset, workspace) dfC = fabric.list_columns(dataset=dataset, workspace=workspace) diff --git a/src/sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py b/src/sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py index 2140e183..e7c7b114 100644 --- a/src/sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py +++ b/src/sempy_labs/directlake/_update_directlake_model_lakehouse_connection.py @@ -5,7 +5,7 @@ resolve_lakehouse_name, resolve_workspace_name_and_id, ) -from sempy_labs._tom import connect_semantic_model +from sempy_labs.tom.model import connect_semantic_model from typing import List, Optional, Union import sempy_labs._icons as icons diff --git a/src/sempy_labs/directlake/_update_directlake_partition_entity.py b/src/sempy_labs/directlake/_update_directlake_partition_entity.py index 471d3c73..3e25956f 100644 --- a/src/sempy_labs/directlake/_update_directlake_partition_entity.py +++ b/src/sempy_labs/directlake/_update_directlake_partition_entity.py @@ -1,5 +1,6 @@ +import sempy import sempy.fabric as fabric -from sempy_labs._tom import connect_semantic_model +from sempy_labs.tom.model import connect_semantic_model from sempy_labs._helper_functions import resolve_lakehouse_name from typing import List, Optional, Union import sempy_labs._icons as icons diff --git a/src/sempy_labs/lakehouse/__init__.py b/src/sempy_labs/lakehouse/__init__.py index 9db8e01a..b643ec3d 100644 --- a/src/sempy_labs/lakehouse/__init__.py +++ b/src/sempy_labs/lakehouse/__init__.py @@ -5,8 +5,7 @@ optimize_lakehouse_tables, ) -from sempy_labs.lakehouse._shortcuts import ( - list_shortcuts, +from sempy_labs.lakehouse._shortcuts import ( # create_shortcut, create_shortcut_onelake, delete_shortcut, @@ -16,8 +15,7 @@ "get_lakehouse_columns", "get_lakehouse_tables", "lakehouse_attached", - "optimize_lakehouse_tables", - "list_shortcuts", + "optimize_lakehouse_tables", # create_shortcut, "create_shortcut_onelake", "delete_shortcut", diff --git a/src/sempy_labs/lakehouse/_get_lakehouse_tables.py b/src/sempy_labs/lakehouse/_get_lakehouse_tables.py index 319191fb..1bdd0206 100644 --- a/src/sempy_labs/lakehouse/_get_lakehouse_tables.py +++ b/src/sempy_labs/lakehouse/_get_lakehouse_tables.py @@ -1,3 +1,4 @@ +import sempy import sempy.fabric as fabric import pandas as pd from pyspark.sql import SparkSession diff --git a/src/sempy_labs/lakehouse/_shortcuts.py b/src/sempy_labs/lakehouse/_shortcuts.py index a02073ed..8d3df98b 100644 --- a/src/sempy_labs/lakehouse/_shortcuts.py +++ b/src/sempy_labs/lakehouse/_shortcuts.py @@ -163,99 +163,6 @@ def create_shortcut( ) -def list_shortcuts( - lakehouse: Optional[str] = None, workspace: Optional[str] = None -) -> pd.DataFrame: - """ - Shows all shortcuts which exist in a Fabric lakehouse. - - Parameters - ---------- - lakehouse : str, default=None - The Fabric lakehouse name. - Defaults to None which resolves to the lakehouse attached to the notebook. - workspace : str, default=None - The name of the Fabric workspace in which lakehouse resides. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - - Returns - ------- - pandas.DataFrame - A pandas dataframe showing all the shortcuts which exist in the specified lakehouse. - """ - - (workspace, workspace_id) = resolve_workspace_name_and_id(workspace) - - if lakehouse == None: - lakehouse_id = fabric.get_lakehouse_id() - lakehouse = resolve_lakehouse_name(lakehouse_id, workspace) - else: - lakehouse_id = resolve_lakehouse_id(lakehouse, workspace) - - df = pd.DataFrame( - columns=[ - "Shortcut Name", - "Shortcut Path", - "Source", - "Source Lakehouse Name", - "Source Workspace Name", - "Source Path", - "Source Connection ID", - "Source Location", - "Source SubPath", - ] - ) - - client = fabric.FabricRestClient() - response = client.get( - f"/v1/workspaces/{workspace_id}/items/{lakehouse_id}/shortcuts" - ) - if response.status_code == 200: - for s in response.json()["value"]: - shortcutName = s["name"] - shortcutPath = s["path"] - source = list(s["target"].keys())[0] - ( - sourceLakehouseName, - sourceWorkspaceName, - sourcePath, - connectionId, - location, - subpath, - ) = (None, None, None, None, None, None) - if source == "oneLake": - sourceLakehouseId = s["target"][source]["itemId"] - sourcePath = s["target"][source]["path"] - sourceWorkspaceId = s["target"][source]["workspaceId"] - sourceWorkspaceName = fabric.resolve_workspace_name(sourceWorkspaceId) - sourceLakehouseName = resolve_lakehouse_name( - sourceLakehouseId, sourceWorkspaceName - ) - else: - connectionId = s["target"][source]["connectionId"] - location = s["target"][source]["location"] - subpath = s["target"][source]["subpath"] - - new_data = { - "Shortcut Name": shortcutName, - "Shortcut Path": shortcutPath, - "Source": source, - "Source Lakehouse Name": sourceLakehouseName, - "Source Workspace Name": sourceWorkspaceName, - "Source Path": sourcePath, - "Source Connection ID": connectionId, - "Source Location": location, - "Source SubPath": subpath, - } - df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) - - print( - f"This function relies on an API which is not yet official as of May 21, 2024. Once the API becomes official this function will work as expected." - ) - return df - - def delete_shortcut( shortcut_name: str, lakehouse: Optional[str] = None, workspace: Optional[str] = None ): diff --git a/src/sempy_labs/migration/__init__.py b/src/sempy_labs/migration/__init__.py index 60f78892..fd36042c 100644 --- a/src/sempy_labs/migration/__init__.py +++ b/src/sempy_labs/migration/__init__.py @@ -14,7 +14,9 @@ ) from sempy_labs.migration._migration_validation import ( migration_validation, - # list_semantic_model_objects +) +from sempy_labs.migration._refresh_calc_tables import ( + refresh_calc_tables, ) __all__ = [ @@ -25,5 +27,5 @@ "migrate_model_objects_to_semantic_model", "migrate_tables_columns_to_semantic_model", "migration_validation", - # list_semantic_model_objects + "refresh_calc_tables" ] diff --git a/src/sempy_labs/migration/_migrate_calctables_to_lakehouse.py b/src/sempy_labs/migration/_migrate_calctables_to_lakehouse.py index a78dd80b..d6400f70 100644 --- a/src/sempy_labs/migration/_migrate_calctables_to_lakehouse.py +++ b/src/sempy_labs/migration/_migrate_calctables_to_lakehouse.py @@ -8,7 +8,7 @@ resolve_lakehouse_id, create_abfss_path, ) -from sempy_labs._tom import connect_semantic_model +from sempy_labs.tom.model import connect_semantic_model from pyspark.sql import SparkSession from typing import List, Optional, Union from sempy._utils._log import log diff --git a/src/sempy_labs/migration/_migrate_calctables_to_semantic_model.py b/src/sempy_labs/migration/_migrate_calctables_to_semantic_model.py index d3cf288c..360d3303 100644 --- a/src/sempy_labs/migration/_migrate_calctables_to_semantic_model.py +++ b/src/sempy_labs/migration/_migrate_calctables_to_semantic_model.py @@ -3,7 +3,7 @@ import re, datetime, time from sempy_labs.lakehouse._get_lakehouse_tables import get_lakehouse_tables from sempy_labs._helper_functions import resolve_lakehouse_name -from sempy_labs._tom import connect_semantic_model +from sempy_labs.tom.model import connect_semantic_model from typing import Optional from sempy._utils._log import log import sempy_labs._icons as icons diff --git a/src/sempy_labs/migration/_migrate_model_objects_to_semantic_model.py b/src/sempy_labs/migration/_migrate_model_objects_to_semantic_model.py index 645a3ceb..9212977d 100644 --- a/src/sempy_labs/migration/_migrate_model_objects_to_semantic_model.py +++ b/src/sempy_labs/migration/_migrate_model_objects_to_semantic_model.py @@ -3,7 +3,7 @@ import re, datetime, time from sempy_labs._list_functions import list_tables from sempy_labs._helper_functions import create_relationship_name -from sempy_labs._tom import connect_semantic_model +from sempy_labs.tom.model import connect_semantic_model from typing import Optional from sempy._utils._log import log import sempy_labs._icons as icons diff --git a/src/sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py b/src/sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py index a7ce514e..fc664d3b 100644 --- a/src/sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py +++ b/src/sempy_labs/migration/_migrate_tables_columns_to_semantic_model.py @@ -6,7 +6,7 @@ from sempy_labs.directlake._get_shared_expression import get_shared_expression from sempy_labs._helper_functions import resolve_lakehouse_name from sempy_labs.lakehouse._lakehouse import lakehouse_attached -from sempy_labs._tom import connect_semantic_model +from sempy_labs.tom.model import connect_semantic_model from typing import List, Optional, Union from sempy._utils._log import log import sempy_labs._icons as icons diff --git a/src/sempy_labs/migration/_migration_validation.py b/src/sempy_labs/migration/_migration_validation.py index e2eb2abd..36f26f1b 100644 --- a/src/sempy_labs/migration/_migration_validation.py +++ b/src/sempy_labs/migration/_migration_validation.py @@ -1,173 +1,10 @@ import sempy import sempy.fabric as fabric import pandas as pd -from sempy_labs._helper_functions import create_relationship_name -from sempy_labs._tom import connect_semantic_model from typing import List, Optional, Union +from sempy_labs._list_functions import list_semantic_model_objects from sempy._utils._log import log - -def list_semantic_model_objects(dataset: str, workspace: Optional[str] = None): - """ - Shows a list of semantic model objects. - - Parameters - ---------- - dataset : str - Name of the semantic model. - workspace : str, default=None - The Fabric workspace name. - Defaults to None which resolves to the workspace of the attached lakehouse - or if no lakehouse attached, resolves to the workspace of the notebook. - - - Returns - ------- - pandas.DataFrame - A pandas dataframe showing a list of objects in the semantic model - """ - - df = pd.DataFrame(columns=["Parent Name", "Object Name", "Object Type"]) - with connect_semantic_model( - dataset=dataset, workspace=workspace, readonly=True - ) as tom: - for t in tom._model.Tables: - if t.CalculationGroup is not None: - new_data = { - "Parent Name": t.Parent.Name, - "Object Name": t.Name, - "Object Type": "Calculation Group", - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - for ci in t.CalculationGroup.CalculationItems: - new_data = { - "Parent Name": t.Name, - "Object Name": ci.Name, - "Object Type": str(ci.ObjectType), - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - elif any(str(p.SourceType) == "Calculated" for p in t.Partitions): - new_data = { - "Parent Name": t.Parent.Name, - "Object Name": t.Name, - "Object Type": "Calculated Table", - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - else: - new_data = { - "Parent Name": t.Parent.Name, - "Object Name": t.Name, - "Object Type": str(t.ObjectType), - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - for c in t.Columns: - if str(c.Type) != "RowNumber": - if str(c.Type) == "Calculated": - new_data = { - "Parent Name": c.Parent.Name, - "Object Name": c.Name, - "Object Type": "Calculated Column", - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - else: - new_data = { - "Parent Name": c.Parent.Name, - "Object Name": c.Name, - "Object Type": str(c.ObjectType), - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - for m in t.Measures: - new_data = { - "Parent Name": m.Parent.Name, - "Object Name": m.Name, - "Object Type": str(m.ObjectType), - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - for h in t.Hierarchies: - new_data = { - "Parent Name": h.Parent.Name, - "Object Name": h.Name, - "Object Type": str(h.ObjectType), - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - for l in h.Levels: - new_data = { - "Parent Name": l.Parent.Name, - "Object Name": l.Name, - "Object Type": str(l.ObjectType), - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - for p in t.Partitions: - new_data = { - "Parent Name": p.Parent.Name, - "Object Name": p.Name, - "Object Type": str(p.ObjectType), - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - for r in tom._model.Relationships: - rName = create_relationship_name( - r.FromTable.Name, r.FromColumn.Name, r.ToTable.Name, r.ToColumn.Name - ) - new_data = { - "Parent Name": r.Parent.Name, - "Object Name": rName, - "Object Type": str(r.ObjectType), - } - df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) - for role in tom._model.Roles: - new_data = { - "Parent Name": role.Parent.Name, - "Object Name": role.Name, - "Object Type": str(role.ObjectType), - } - df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) - for rls in role.TablePermissions: - new_data = { - "Parent Name": role.Name, - "Object Name": rls.Name, - "Object Type": str(rls.ObjectType), - } - df = pd.concat( - [df, pd.DataFrame(new_data, index=[0])], ignore_index=True - ) - for tr in tom._model.Cultures: - new_data = { - "Parent Name": tr.Parent.Name, - "Object Name": tr.Name, - "Object Type": str(tr.ObjectType), - } - df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) - for per in tom._model.Perspectives: - new_data = { - "Parent Name": per.Parent.Name, - "Object Name": per.Name, - "Object Type": str(per.ObjectType), - } - df = pd.concat([df, pd.DataFrame(new_data, index=[0])], ignore_index=True) - - return df - - @log def migration_validation( dataset: str, diff --git a/src/sempy_labs/migration/_refresh_calc_tables.py b/src/sempy_labs/migration/_refresh_calc_tables.py index 99a714e1..8851d38d 100644 --- a/src/sempy_labs/migration/_refresh_calc_tables.py +++ b/src/sempy_labs/migration/_refresh_calc_tables.py @@ -3,7 +3,7 @@ import pandas as pd import re, datetime, time from pyspark.sql import SparkSession -from sempy_labs._tom import connect_semantic_model +from sempy_labs.tom.model import connect_semantic_model from typing import List, Optional, Union from sempy._utils._log import log import sempy_labs._icons as icons diff --git a/src/sempy_labs/report/__init__.py b/src/sempy_labs/report/__init__.py index 51e905f8..4835a10b 100644 --- a/src/sempy_labs/report/__init__.py +++ b/src/sempy_labs/report/__init__.py @@ -1,6 +1,6 @@ from sempy_labs.report._generate_report import ( create_report_from_reportjson, - update_report_from_reportjson, + #update_report_from_reportjson, ) from sempy_labs.report._report_functions import ( get_report_json, @@ -20,7 +20,7 @@ __all__ = [ "create_report_from_reportjson", - "update_report_from_reportjson", + #"update_report_from_reportjson", "get_report_json", # report_dependency_tree, "export_report", diff --git a/src/sempy_labs/tom/__init__.py b/src/sempy_labs/tom/__init__.py new file mode 100644 index 00000000..01161e65 --- /dev/null +++ b/src/sempy_labs/tom/__init__.py @@ -0,0 +1,6 @@ +from sempy_labs.tom.model import TOMWrapper, connect_semantic_model + +__all__ = [ + "TOMWrapper", + "connect_semantic_model" +] \ No newline at end of file diff --git a/src/sempy_labs/_tom.py b/src/sempy_labs/tom/model.py similarity index 99% rename from src/sempy_labs/_tom.py rename to src/sempy_labs/tom/model.py index eb017860..10c3fbe9 100644 --- a/src/sempy_labs/_tom.py +++ b/src/sempy_labs/tom/model.py @@ -19,7 +19,7 @@ class TOMWrapper: """ - Convenience wrapper around the TOM object model for a semantic model. Always use connect_semantic_model function to make sure the TOM object is initialized correctly. + Convenience wrapper around the TOM object model for a semantic model. Always use the connect_semantic_model function to make sure the TOM object is initialized correctly. `XMLA read/write endpoints `_ must be enabled if setting the readonly parameter to False. """ @@ -1937,14 +1937,7 @@ def is_date_table(self, table_name: str): """ import Microsoft.AnalysisServices.Tabular as TOM - isDateTable = False - t = self._model.Tables[table_name] - - if t.DataCategory == "Time": - if any(c.IsKey and c.DataType == TOM.DataType.DateTime for c in t.Columns): - isDateTable = True - - return isDateTable + return any(c.IsKey and c.DataType == TOM.DataType.DateTime for c in self.all_columns() if c.Parent.Name == table_name and c.Parent.DataCategory == 'Time') def mark_as_date_table(self, table_name: str, column_name: str): """ @@ -2010,13 +2003,7 @@ def has_aggs(self): Indicates if the semantic model has any aggregations. """ - hasAggs = False - - for c in self.all_columns(): - if c.AlterateOf is not None: - hasAggs = True - - return hasAggs + return any(c.AlternateOf is not None for c in self.all_columns()) def is_agg_table(self, table_name: str): """ @@ -2048,15 +2035,9 @@ def has_hybrid_table(self): ------- bool Indicates if the semantic model has a hybrid table. - """ + """ - hasHybridTable = False - - for t in self._model.Tables: - if self.is_hybrid_table(table_name=t.Name): - hasHybridTable = True - - return hasHybridTable + return any(self.is_hybrid_table(table_name = t.Name) for t in self._model.Tables) def has_date_table(self): """ @@ -2071,13 +2052,7 @@ def has_date_table(self): Indicates if the semantic model has a table marked as a date table. """ - hasDateTable = False - - for t in self._model.Tables: - if self.is_date_table(table_name=t.Name): - hasDateTable = True - - return hasDateTable + return any(self.is_date_table(table_name = t.Name) for t in self._model.Tables) def is_direct_lake(self): """ @@ -2091,6 +2066,7 @@ def is_direct_lake(self): bool Indicates if the semantic model is in Direct Lake mode. """ + import Microsoft.AnalysisServices.Tabular as TOM return any( p.Mode == TOM.ModeType.DirectLake