From b2d6ce3a764902bae9467dd7ce2ddede2b08ef01 Mon Sep 17 00:00:00 2001 From: Afroz Alam Date: Mon, 26 Feb 2024 22:04:45 +0000 Subject: [PATCH] SNOW-1039216: add default memory pool for pyarrow (#1879) * SNOW-1039216: add default memory pool for pyarrow * SNOW-1039216: add docstring; don't change default value if already set --- src/snowflake/connector/options.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/snowflake/connector/options.py b/src/snowflake/connector/options.py index bf867a6c0..6aea0ee34 100644 --- a/src/snowflake/connector/options.py +++ b/src/snowflake/connector/options.py @@ -5,6 +5,7 @@ from __future__ import annotations import importlib +import os import warnings from importlib.metadata import distributions from logging import getLogger @@ -79,6 +80,10 @@ def _import_or_missing_pandas_option() -> ( pyarrow = importlib.import_module("pyarrow") + # set default memory pool to system for pyarrow to_pandas conversion + if "ARROW_DEFAULT_MEMORY_POOL" not in os.environ: + os.environ["ARROW_DEFAULT_MEMORY_POOL"] = "system" + # Check whether we have the currently supported pyarrow installed installed_packages = { package.metadata["Name"]: package for package in distributions()