ML-2296: Add support for datastore profiles

integration/test_filesystems_integration.py

@@ -22,6 +22,7 @@
 import pandas as pd
 import pytest
 import v3io
+from fsspec.implementations.local import LocalFileSystem
 
 from integration.integration_test_utils import V3ioHeaders, _generate_table_name
 from storey import (
@@ -38,7 +39,7 @@
  build_flow,
 )
 from storey.dtypes import V3ioError
-from storey.utils import get_remaining_path
+from storey.utils import get_remaining_path, url_to_file_system
 
 
 @pytest.fixture()
@@ -672,3 +673,10 @@ def test_get_path_utils():
  schema, path = get_remaining_path(url)
  assert path == "mycontainer/path/to/object.csv"
  assert schema == "wasbs"
+
+
+def test_ds_get_path_utils():
+ url = "ds://:file@profile/path/to/object.csv"
+ fs, path = url_to_file_system(url, "")
+ assert path == "/path/to/object.csv"
+ assert isinstance(fs, LocalFileSystem)

storey/utils.py

@@ -127,7 +127,7 @@ def get_remaining_path(url):
  if "://" in url:
  parsed_url = urlparse(url)
  scheme = parsed_url.scheme.lower()
- if scheme in ("v3io", "dbfs"):
+ if scheme in ("ds", "v3io", "dbfs"):
  remaining_path = parsed_url.path
  elif scheme in ["wasb", "wasbs"]:
  remaining_path = f"{parsed_url.username}{parsed_url.path}"
@@ -138,6 +138,12 @@ def get_remaining_path(url):
 
 def url_to_file_system(url, storage_options):
  scheme, remaining_path = get_remaining_path(url)
+ if url.startswith("ds://"):
+ parsed_url = urlparse(url)
+ if parsed_url.password:
+ scheme = parsed_url.password
+ else:
+ raise ValueError("Datastore profile URL is expected to have underlying scheme embedded as password")
  if scheme:
  load_fs_dependencies(scheme)