config set get

doc/source/user_guide/config.rst

@@ -0,0 +1,54 @@
+.. _config:
+
+=============
+Configuration
+=============
+
+In Xorbits, there are two types of configuration and option setting approaches: 
+
+- cluster-level: applied to the whole cluster when starting the supervisor or the workers.
+- job-level: applied to a specific Xorbits job or Python script.
+
+Cluster-Level Configuration
+---------------------------
+
+Cluster-level configurations are applied to the entire Xorbits cluster and affect all jobs 
+running on it. These settings are typically defined when starting the Xorbits cluster 
+(i.e., the supervisor or the workers) and remain constant throughout the cluster's lifetime.
+
+Examples of cluster-level configurations include:
+
+- Network: use TCP Socket or UCX.
+- Storage: use Shared Memory or Filesystem.
+
+These configurations are usually set through command-line arguments or configuration files 
+when launching the Xorbits cluster.
+
+Job-Level Configuration
+-----------------------
+
+Job-level configurations are specific to individual Xorbits jobs or sessions. These settings allow users to fine-tune the behavior of their specific workloads without affecting other jobs running on the same cluster.
+
+Job-level configurations can be set using the following methods:
+
+1. Using `xorbits.set_option()`:
+
+ .. code-block:: python
+
+ import xorbits.pandas as xpd
+ 
+ xpd.set_option("chunk_store_limit", 1024 ** 3) # Set chunk store limit to 1 GB
+
+2. Using `xorbits.option_context()`:
+
+ .. code-block:: python
+
+ import xorbits.pandas as xpd
+
+ with xpd.option_context({"chunk_store_limit": 1024 ** 3}):
+ # Your Xorbits code here
+ # The chunk_store_limit will be set to 1 GB only within this context
+
+These job-level configurations allow users to optimize their workloads for specific requirements without impacting other jobs or the overall cluster configuration.
+
+By providing both cluster-level and job-level configuration options, Xorbits offers flexibility in managing resources and optimizing performance for various use cases and workloads.

python/xorbits/_mars/config.py

@@ -21,7 +21,7 @@
 import warnings
 from copy import deepcopy
 from functools import reduce
-from typing import Dict, Union
+from typing import Any, Dict, Union
 
 _DEFAULT_REDIRECT_WARN = (
  "Option {source} has been replaced by {target} and "
@@ -244,6 +244,42 @@ def copy(self):
  new_options = Config(deepcopy(self._config))
  return new_options
 
+ def get_option(self, option: str) -> Any:
+ splits = option.split(".")
+ conf = self._config
+ for name in splits[:-1]:
+ config = conf.get(name)
+ if not isinstance(config, dict):
+ raise AttributeError(f"No such keys(s): {option}.")
+ else:
+ conf = config
+
+ key = splits[-1]
+ if key not in conf:
+ raise AttributeError(f"No such keys(s): {option}.")
+ (value, _) = conf.get(key)
+ return value
+
+ def set_option(self, option: str, value: Any) -> Any:
+ splits = option.split(".")
+ conf = self._config
+ for name in splits[:-1]:
+ config = conf.get(name)
+ if not isinstance(config, dict):
+ raise AttributeError(f"No such keys(s): {option}.")
+ else:
+ conf = config
+
+ key = splits[-1]
+ if key not in conf:
+ raise AttributeError(f"No such keys(s): {option}.")
+ (old_value, validator) = conf.get(key)
+ if validator is not None:
+ if not validator(value):
+ raise ValueError(f"Invalid value {value} for option {option}")
+
+ conf[key] = value, validator
+
  def update(self, new_config: Union["Config", Dict]):
  if not isinstance(new_config, dict):
  new_config = new_config._config

python/xorbits/_mars/tests/test_config.py

@@ -103,6 +103,25 @@ def test_config_copy():
  assert target_cfg.a.b.c == 1
 
 
+def test_get_set_config():
+ cfg = Config()
+ cfg.register_option("a.b.c", 1, validator=is_integer)
+
+ assert cfg.get_option("a.b.c") == 1
+
+ with pytest.raises(AttributeError):
+ cfg.get_option("non-exist")
+
+ cfg.set_option("a.b.c", 2)
+ assert cfg.get_option("a.b.c") == 2
+
+ with pytest.raises(ValueError):
+ cfg.set_option("a.b.c", "foo")
+
+ with pytest.raises(AttributeError):
+ cfg.set_option("non-exist", "foo")
+
+
 def test_pickle_config():
  cfg = Config()
  cfg.register_option("a.b.c", 1)

python/xorbits/pandas/_config/config.py

@@ -17,34 +17,36 @@
 
 import pandas as pd
 
-from ..._mars.config import option_context as mars_option_context
-from ..._mars.config import options
+from ..._mars.config import option_context as xorbits_option_context
+from ..._mars.config import options as xorbits_options
 from ...core.utils.docstring import attach_module_callable_docstring
 
 
 def set_option(pat: Any, value: Any) -> None:
  try:
  attr_list = pat.split(".")
  if len(attr_list) == 1:
- getattr(options, attr_list[0])
- setattr(options, attr_list[0], value)
+ getattr(xorbits_options, attr_list[0])
+ setattr(xorbits_options, attr_list[0], value)
  else:
- setattr(reduce(getattr, attr_list[:-1], options), attr_list[-1], value)
+ setattr(
+ reduce(getattr, attr_list[:-1], xorbits_options), attr_list[-1], value
+ )
  except:
  pd.set_option(pat, value)
 
 
 def get_option(pat: Any) -> Any:
  try:
  attr_list = pat.split(".")
- return reduce(getattr, attr_list, options)
+ return reduce(getattr, attr_list, xorbits_options)
  except:
  return pd.get_option(pat)
 
 
 def reset_option(pat) -> None:
  try:
- options.reset_option(pat)
+ xorbits_options.reset_option(pat)
  except:
  pd.reset_option(pat)
 
@@ -57,19 +59,24 @@ class option_context:
  def __init__(self, *args):
  # convert tuple to dict
  context_dict = dict(args[i : i + 2] for i in range(0, len(args), 2))
- mars_dict = {}
+ xorbits_dict = {}
  pd_dict = {}
  for key, value in context_dict.items():
  try:
  pd.get_option(key)
  pd_dict[key] = value
- except:
- mars_dict[key] = value
-
+ except pd._config.config.OptionError:
+ try:
+ xorbits_options.get_option(key)
+ xorbits_dict[key] = value
+ except AttributeError:
+ raise AttributeError(
+ f"No such keys(s): '{key}' in pandas and xorbits."
+ )
  self.option_contexts = None
  self.pandas_option_context = None
- if mars_dict:
- self.option_contexts = mars_option_context(mars_dict)
+ if xorbits_dict:
+ self.option_contexts = xorbits_option_context(xorbits_dict)
  if pd_dict:
  self.pandas_option_context = pd.option_context(
  *tuple(item for sublist in pd_dict.items() for item in sublist)

python/xorbits/pandas/_config/test/test_config.py

@@ -61,16 +61,16 @@ def test_options_execute(setup):
  xpd.set_option("display.max_rows", "100")
 
  # test error option
- with pytest.raises(pd._config.config.OptionError):
+ with pytest.raises(AttributeError):
  xpd.set_option("non-exist", None)
 
- with pytest.raises(pd._config.config.OptionError):
+ with pytest.raises(AttributeError):
  xpd.get_option("non-exist")
 
- with pytest.raises(pd._config.config.OptionError):
+ with pytest.raises(AttributeError):
  xpd.reset_option("non-exist")
 
- with pytest.raises(ValueError):
+ with pytest.raises(AttributeError):
  xpd.option_context("non-exist", 100)
 
  # test invalid type