diff --git a/.gitignore b/.gitignore index b9ab6843fe40..a055cde851a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.iml *.so *.pyd +*.pdb *.ipynb .ENV .env diff --git a/Cargo.lock b/Cargo.lock index 19b1b0fd8c26..cb633d353da2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2568,9 +2568,9 @@ dependencies = [ [[package]] name = "numpy" -version = "0.22.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb929bc0da91a4d85ed6c0a84deaa53d411abfb387fc271124f91bf6b89f14e" +checksum = "b94caae805f998a07d33af06e6a3891e38556051b8045c615470a71590e13e78" dependencies = [ "libc", "ndarray", @@ -2578,7 +2578,7 @@ dependencies = [ "num-integer", "num-traits", "pyo3", - "rustc-hash 1.1.0", + "rustc-hash", ] [[package]] @@ -3673,9 +3673,8 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +version = "0.23.3" +source = "git+https://github.com/bschoenmaeckers/pyo3.git?branch=release-0.23#f209b1a87708a03f2234b744bb64a9f9825ec768" dependencies = [ "cfg-if", "chrono", @@ -3693,9 +3692,8 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +version = "0.23.3" +source = "git+https://github.com/bschoenmaeckers/pyo3.git?branch=release-0.23#f209b1a87708a03f2234b744bb64a9f9825ec768" dependencies = [ "once_cell", "target-lexicon", @@ -3703,9 +3701,8 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +version = "0.23.3" +source = "git+https://github.com/bschoenmaeckers/pyo3.git?branch=release-0.23#f209b1a87708a03f2234b744bb64a9f9825ec768" dependencies = [ "libc", "pyo3-build-config", @@ -3713,9 +3710,8 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +version = "0.23.3" +source = "git+https://github.com/bschoenmaeckers/pyo3.git?branch=release-0.23#f209b1a87708a03f2234b744bb64a9f9825ec768" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -3725,9 +3721,8 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +version = "0.23.3" +source = "git+https://github.com/bschoenmaeckers/pyo3.git?branch=release-0.23#f209b1a87708a03f2234b744bb64a9f9825ec768" dependencies = [ "heck", "proc-macro2", @@ -3773,7 +3768,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.0", + "rustc-hash", "rustls 0.23.19", "socket2", "thiserror 2.0.3", @@ -3791,7 +3786,7 @@ dependencies = [ "getrandom", "rand", "ring", - "rustc-hash 2.1.0", + "rustc-hash", "rustls 0.23.19", "rustls-pki-types", "slab", @@ -4095,12 +4090,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.0" diff --git a/Cargo.toml b/Cargo.toml index a713d08d7553..a431283c497f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,13 +56,13 @@ memmap = { package = "memmap2", version = "0.9" } multiversion = "0.7" ndarray = { version = "0.16", default-features = false } num-traits = "0.2" -numpy = "0.22" +numpy = "0.23" object_store = { version = "0.11", default-features = false } once_cell = "1" parking_lot = "0.12" percent-encoding = "2.3" pin-project-lite = "0.2" -pyo3 = "0.22" +pyo3 = { git = "https://github.com/bschoenmaeckers/pyo3.git", branch = "release-0.23" } rand = "0.8" rand_distr = "0.4" raw-cpuid = "11" @@ -136,6 +136,8 @@ features = [ [patch.crates-io] # packed_simd_2 = { git = "https://github.com/rust-lang/packed_simd", rev = "e57c7ba11386147e6d2cbad7c88f376aab4bdc86" } # simd-json = { git = "https://github.com/ritchie46/simd-json", branch = "alignment" } +pyo3 = { git = "https://github.com/bschoenmaeckers/pyo3.git", branch = "release-0.23" } +pyo3-ffi = { git = "https://github.com/bschoenmaeckers/pyo3.git", branch = "release-0.23" } [profile.mindebug-dev] inherits = "dev" diff --git a/crates/polars-error/src/lib.rs b/crates/polars-error/src/lib.rs index 5810c28160de..c19e90ec20b3 100644 --- a/crates/polars-error/src/lib.rs +++ b/crates/polars-error/src/lib.rs @@ -3,6 +3,7 @@ mod warning; use std::borrow::Cow; use std::collections::TryReserveError; +use std::convert::Infallible; use std::error::Error; use std::fmt::{self, Display, Formatter, Write}; use std::ops::Deref; @@ -168,6 +169,12 @@ impl From for PolarsError { } } +impl From for PolarsError { + fn from(_: Infallible) -> Self { + unreachable!() + } +} + pub type PolarsResult = Result; impl PolarsError { diff --git a/crates/polars-mem-engine/src/executors/scan/python_scan.rs b/crates/polars-mem-engine/src/executors/scan/python_scan.rs index f74da737d0ac..b8d4ca9e3e7f 100644 --- a/crates/polars-mem-engine/src/executors/scan/python_scan.rs +++ b/crates/polars-mem-engine/src/executors/scan/python_scan.rs @@ -2,8 +2,8 @@ use polars_core::error::to_compute_err; use polars_core::utils::accumulate_dataframes_vertical; use pyo3::exceptions::PyStopIteration; use pyo3::prelude::*; -use pyo3::types::PyBytes; -use pyo3::{intern, PyTypeInfo}; +use pyo3::types::{PyBytes, PyNone}; +use pyo3::{intern, IntoPyObjectExt, PyTypeInfo}; use super::*; @@ -41,7 +41,7 @@ impl Executor for PythonScanExec { let with_columns = self.options.with_columns.take(); let n_rows = self.options.n_rows.take(); Python::with_gil(|py| { - let pl = PyModule::import_bound(py, intern!(py, "polars")).unwrap(); + let pl = PyModule::import(py, intern!(py, "polars")).unwrap(); let utils = pl.getattr(intern!(py, "_utils")).unwrap(); let callable = utils.getattr(intern!(py, "_execute_from_rust")).unwrap(); @@ -50,14 +50,14 @@ impl Executor for PythonScanExec { let with_columns = with_columns.map(|cols| cols.iter().cloned().collect::>()); let predicate = match &self.options.predicate { - PythonPredicate::PyArrow(s) => s.into_py(py), - PythonPredicate::None => None::<()>.into_py(py), + PythonPredicate::PyArrow(s) => s.into_bound_py_any(py).unwrap(), + PythonPredicate::None => None::<()>.into_bound_py_any(py).unwrap(), PythonPredicate::Polars(_) => { assert!(self.predicate.is_some(), "should be set"); match &self.predicate_serialized { - None => None::<()>.into_py(py), - Some(buf) => PyBytes::new_bound(py, buf).to_object(py), + None => PyNone::get(py).to_owned().into_any(), + Some(buf) => PyBytes::new(py, buf).into_any(), } }, }; @@ -125,7 +125,7 @@ impl Executor for PythonScanExec { } chunks.push(df) }, - Err(err) if err.matches(py, PyStopIteration::type_object_bound(py)) => break, + Err(err) if err.matches(py, PyStopIteration::type_object(py))? => break, Err(err) => { polars_bail!(ComputeError: "caught exception during execution of a Python source, exception: {}", err) }, diff --git a/crates/polars-plan/src/dsl/python_udf.rs b/crates/polars-plan/src/dsl/python_udf.rs index cd133ceb646e..cb7b834627a2 100644 --- a/crates/polars-plan/src/dsl/python_udf.rs +++ b/crates/polars-plan/src/dsl/python_udf.rs @@ -78,11 +78,11 @@ impl PythonUdfExpression { // Load UDF Python::with_gil(|py| { - let pickle = PyModule::import_bound(py, "pickle") + let pickle = PyModule::import(py, "pickle") .expect("unable to import 'pickle'") .getattr("loads") .unwrap(); - let arg = (PyBytes::new_bound(py, remainder),); + let arg = (PyBytes::new(py, remainder),); let python_function = pickle.call1(arg).map_err(from_pyerr)?; Ok(Arc::new(Self::new( python_function.into(), @@ -136,7 +136,7 @@ impl ColumnsUdf for PythonUdfExpression { Python::with_gil(|py| { // Try pickle to serialize the UDF, otherwise fall back to cloudpickle. - let pickle = PyModule::import_bound(py, "pickle") + let pickle = PyModule::import(py, "pickle") .expect("unable to import 'pickle'") .getattr("dumps") .unwrap(); @@ -144,7 +144,7 @@ impl ColumnsUdf for PythonUdfExpression { let (dumped, use_cloudpickle) = match pickle_result { Ok(dumped) => (dumped, false), Err(_) => { - let cloudpickle = PyModule::import_bound(py, "cloudpickle") + let cloudpickle = PyModule::import(py, "cloudpickle") .map_err(from_pyerr)? .getattr("dumps") .unwrap(); diff --git a/crates/polars-python/src/cloud.rs b/crates/polars-python/src/cloud.rs index 19d4f6dfda07..08379da8e955 100644 --- a/crates/polars-python/src/cloud.rs +++ b/crates/polars-python/src/cloud.rs @@ -14,11 +14,11 @@ use crate::lazyframe::visit::NodeTraverser; use crate::{PyDataFrame, PyLazyFrame}; #[pyfunction] -pub fn prepare_cloud_plan(lf: PyLazyFrame, py: Python) -> PyResult { +pub fn prepare_cloud_plan(lf: PyLazyFrame, py: Python<'_>) -> PyResult> { let plan = lf.ldf.logical_plan; let bytes = polars::prelude::prepare_cloud_plan(plan).map_err(PyPolarsErr::from)?; - Ok(PyBytes::new_bound(py, &bytes).to_object(py)) + Ok(PyBytes::new(py, &bytes)) } /// Take a serialized `IRPlan` and execute it on the GPU engine. @@ -62,13 +62,13 @@ fn gpu_post_opt( expr_arena: &mut Arena, ) -> PolarsResult<()> { // Get cuDF Python function. - let cudf = PyModule::import_bound(py, intern!(py, "cudf_polars")).unwrap(); + let cudf = PyModule::import(py, intern!(py, "cudf_polars")).unwrap(); let lambda = cudf.getattr(intern!(py, "execute_with_cudf")).unwrap(); // Define cuDF config. - let polars = PyModule::import_bound(py, intern!(py, "polars")).unwrap(); + let polars = PyModule::import(py, intern!(py, "polars")).unwrap(); let engine = polars.getattr(intern!(py, "GPUEngine")).unwrap(); - let kwargs = [("raise_on_fail", true)].into_py_dict_bound(py); + let kwargs = [("raise_on_fail", true)].into_py_dict(py).unwrap(); let engine = engine.call((), Some(&kwargs)).unwrap(); // Define node traverser. @@ -79,7 +79,7 @@ fn gpu_post_opt( // Pass the node visitor which allows the Python callback to replace parts of the query plan. // Remove "cuda" or specify better once we have multiple post-opt callbacks. - let kwargs = [("config", engine)].into_py_dict_bound(py); + let kwargs = [("config", engine)].into_py_dict(py).unwrap(); lambda .call((nt,), Some(&kwargs)) .map_err(|e| polars_err!(ComputeError: "'cuda' conversion failed: {}", e))?; diff --git a/crates/polars-python/src/conversion/any_value.rs b/crates/polars-python/src/conversion/any_value.rs index 98c2846506d4..e986ab17fc09 100644 --- a/crates/polars-python/src/conversion/any_value.rs +++ b/crates/polars-python/src/conversion/any_value.rs @@ -10,29 +10,37 @@ use polars_core::export::chrono::{NaiveDate, NaiveDateTime, NaiveTime, TimeDelta use polars_core::utils::any_values_to_supertype_and_n_dtypes; use polars_core::utils::arrow::temporal_conversions::date32_to_date; use pyo3::exceptions::{PyOverflowError, PyTypeError, PyValueError}; -use pyo3::intern; use pyo3::prelude::*; use pyo3::types::{ PyBool, PyBytes, PyDict, PyFloat, PyInt, PyList, PySequence, PyString, PyTuple, PyType, }; +use pyo3::{intern, IntoPyObjectExt}; use super::datetime::{ elapsed_offset_to_timedelta, nanos_since_midnight_to_naivetime, timestamp_to_naive_datetime, }; use super::{decimal_to_digits, struct_dict, ObjectValue, Wrap}; use crate::error::PyPolarsErr; -use crate::py_modules::{SERIES, UTILS}; +use crate::py_modules::{pl_series, pl_utils}; use crate::series::PySeries; -impl IntoPy for Wrap> { - fn into_py(self, py: Python) -> PyObject { +impl<'py> IntoPyObject<'py> for Wrap> { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { any_value_into_py_object(self.0, py) } } -impl ToPyObject for Wrap> { - fn to_object(&self, py: Python) -> PyObject { - self.clone().into_py(py) +impl<'py> IntoPyObject<'py> for &Wrap> { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + self.clone().into_pyobject(py) } } @@ -42,31 +50,34 @@ impl<'py> FromPyObject<'py> for Wrap> { } } -pub(crate) fn any_value_into_py_object(av: AnyValue, py: Python) -> PyObject { - let utils = UTILS.bind(py); +pub(crate) fn any_value_into_py_object<'py>( + av: AnyValue, + py: Python<'py>, +) -> PyResult> { + let utils = pl_utils(py).bind(py); match av { - AnyValue::UInt8(v) => v.into_py(py), - AnyValue::UInt16(v) => v.into_py(py), - AnyValue::UInt32(v) => v.into_py(py), - AnyValue::UInt64(v) => v.into_py(py), - AnyValue::Int8(v) => v.into_py(py), - AnyValue::Int16(v) => v.into_py(py), - AnyValue::Int32(v) => v.into_py(py), - AnyValue::Int64(v) => v.into_py(py), - AnyValue::Int128(v) => v.into_py(py), - AnyValue::Float32(v) => v.into_py(py), - AnyValue::Float64(v) => v.into_py(py), - AnyValue::Null => py.None(), - AnyValue::Boolean(v) => v.into_py(py), - AnyValue::String(v) => v.into_py(py), - AnyValue::StringOwned(v) => v.into_py(py), + AnyValue::UInt8(v) => v.into_bound_py_any(py), + AnyValue::UInt16(v) => v.into_bound_py_any(py), + AnyValue::UInt32(v) => v.into_bound_py_any(py), + AnyValue::UInt64(v) => v.into_bound_py_any(py), + AnyValue::Int8(v) => v.into_bound_py_any(py), + AnyValue::Int16(v) => v.into_bound_py_any(py), + AnyValue::Int32(v) => v.into_bound_py_any(py), + AnyValue::Int64(v) => v.into_bound_py_any(py), + AnyValue::Int128(v) => v.into_bound_py_any(py), + AnyValue::Float32(v) => v.into_bound_py_any(py), + AnyValue::Float64(v) => v.into_bound_py_any(py), + AnyValue::Null => py.None().into_bound_py_any(py), + AnyValue::Boolean(v) => v.into_bound_py_any(py), + AnyValue::String(v) => v.into_bound_py_any(py), + AnyValue::StringOwned(v) => v.into_bound_py_any(py), AnyValue::Categorical(idx, rev, arr) | AnyValue::Enum(idx, rev, arr) => { let s = if arr.is_null() { rev.get(idx) } else { unsafe { arr.deref_unchecked().value(idx as usize) } }; - s.into_py(py) + s.into_bound_py_any(py) }, AnyValue::CategoricalOwned(idx, rev, arr) | AnyValue::EnumOwned(idx, rev, arr) => { let s = if arr.is_null() { @@ -74,11 +85,11 @@ pub(crate) fn any_value_into_py_object(av: AnyValue, py: Python) -> PyObject { } else { unsafe { arr.deref_unchecked().value(idx as usize) } }; - s.into_py(py) + s.into_bound_py_any(py) }, AnyValue::Date(v) => { let date = date32_to_date(v); - date.into_py(py) + date.into_bound_py_any(py) }, AnyValue::Datetime(v, time_unit, time_zone) => { datetime_to_py_object(py, utils, v, time_unit, time_zone) @@ -92,26 +103,30 @@ pub(crate) fn any_value_into_py_object(av: AnyValue, py: Python) -> PyObject { ), AnyValue::Duration(v, time_unit) => { let time_delta = elapsed_offset_to_timedelta(v, time_unit); - time_delta.into_py(py) + time_delta.into_bound_py_any(py) + }, + AnyValue::Time(v) => nanos_since_midnight_to_naivetime(v).into_bound_py_any(py), + AnyValue::Array(v, _) | AnyValue::List(v) => PySeries::new(v).to_list(py), + ref av @ AnyValue::Struct(_, _, flds) => { + Ok(struct_dict(py, av._iter_struct_av(), flds)?.into_any()) + }, + AnyValue::StructOwned(payload) => { + Ok(struct_dict(py, payload.0.into_iter(), &payload.1)?.into_any()) }, - AnyValue::Time(v) => nanos_since_midnight_to_naivetime(v).into_py(py), - AnyValue::Array(v, _) | AnyValue::List(v) => PySeries::new(v).to_list(), - ref av @ AnyValue::Struct(_, _, flds) => struct_dict(py, av._iter_struct_av(), flds), - AnyValue::StructOwned(payload) => struct_dict(py, payload.0.into_iter(), &payload.1), #[cfg(feature = "object")] AnyValue::Object(v) => { let object = v.as_any().downcast_ref::().unwrap(); - object.inner.clone_ref(py) + Ok(object.inner.clone_ref(py).into_bound(py)) }, #[cfg(feature = "object")] AnyValue::ObjectOwned(v) => { let object = v.0.as_any().downcast_ref::().unwrap(); - object.inner.clone_ref(py) + Ok(object.inner.clone_ref(py).into_bound(py)) }, - AnyValue::Binary(v) => PyBytes::new_bound(py, v).into_py(py), - AnyValue::BinaryOwned(v) => PyBytes::new_bound(py, &v).into_py(py), + AnyValue::Binary(v) => PyBytes::new(py, v).into_bound_py_any(py), + AnyValue::BinaryOwned(v) => PyBytes::new(py, &v).into_bound_py_any(py), AnyValue::Decimal(v, scale) => { - let convert = utils.getattr(intern!(py, "to_py_decimal")).unwrap(); + let convert = utils.getattr(intern!(py, "to_py_decimal"))?; const N: usize = 3; let mut buf = [0_u128; N]; let n_digits = decimal_to_digits(v.abs(), &mut buf); @@ -121,22 +136,19 @@ pub(crate) fn any_value_into_py_object(av: AnyValue, py: Python) -> PyObject { N * size_of::(), ) }; - let digits = PyTuple::new_bound(py, buf.iter().take(n_digits)); - convert - .call1((v.is_negative() as u8, digits, n_digits, -(scale as i32))) - .unwrap() - .into_py(py) + let digits = PyTuple::new(py, buf.iter().take(n_digits))?; + convert.call1((v.is_negative() as u8, digits, n_digits, -(scale as i32))) }, } } -fn datetime_to_py_object( - py: Python, - utils: &Bound, +fn datetime_to_py_object<'py>( + py: Python<'py>, + utils: &Bound<'py, PyAny>, v: i64, tu: TimeUnit, tz: Option<&TimeZone>, -) -> PyObject { +) -> PyResult> { if let Some(time_zone) = tz { // When https://github.com/pola-rs/polars/issues/16199 is // implemented, we'll switch to something like: @@ -144,14 +156,11 @@ fn datetime_to_py_object( // let tz: chrono_tz::Tz = time_zone.parse().unwrap(); // let datetime = tz.from_local_datetime(&naive_datetime).earliest().unwrap(); // datetime.into_py(py) - let convert = utils.getattr(intern!(py, "to_py_datetime")).unwrap(); + let convert = utils.getattr(intern!(py, "to_py_datetime"))?; let time_unit = tu.to_ascii(); - convert - .call1((v, time_unit, time_zone.as_str())) - .unwrap() - .into_py(py) + convert.call1((v, time_unit, time_zone.as_str())) } else { - timestamp_to_naive_datetime(v, tu).into_py(py) + timestamp_to_naive_datetime(v, tu).into_pyobject(py) } } @@ -267,7 +276,7 @@ pub(crate) fn py_object_to_any_value<'py>( // Probably needs to wait for // https://github.com/pola-rs/polars/issues/16199 to do it a faster way. Python::with_gil(|py| { - let date = UTILS + let date = pl_utils(py) .bind(py) .getattr(intern!(py, "datetime_to_int")) .unwrap() @@ -352,9 +361,9 @@ pub(crate) fn py_object_to_any_value<'py>( // This constructor is able to go via dedicated type constructors // so it can be much faster. let py = ob.py(); - let kwargs = PyDict::new_bound(py); + let kwargs = PyDict::new(py); kwargs.set_item("strict", strict)?; - let s = SERIES.call_bound(py, (ob,), Some(&kwargs))?; + let s = pl_series(py).call(py, (ob,), Some(&kwargs))?; get_list_from_series(s.bind(py), strict) } @@ -366,10 +375,10 @@ pub(crate) fn py_object_to_any_value<'py>( } else if ob.is_instance_of::() | ob.is_instance_of::() { const INFER_SCHEMA_LENGTH: usize = 25; - let list = ob.downcast::().unwrap(); + let list = ob.downcast::()?; let mut avs = Vec::with_capacity(INFER_SCHEMA_LENGTH); - let mut iter = list.iter()?; + let mut iter = list.try_iter()?; let mut items = Vec::with_capacity(INFER_SCHEMA_LENGTH); for item in (&mut iter).take(INFER_SCHEMA_LENGTH) { items.push(item?); @@ -469,7 +478,7 @@ pub(crate) fn py_object_to_any_value<'py>( Ok(get_struct) } else { let ob_type = ob.get_type(); - let type_name = ob_type.qualname().unwrap().to_string(); + let type_name = ob_type.qualname()?.to_string(); match type_name.as_str() { // Can't use pyo3::types::PyDateTime with abi3-py37 feature, // so need this workaround instead of `isinstance(ob, datetime)`. @@ -488,10 +497,9 @@ pub(crate) fn py_object_to_any_value<'py>( } // Support custom subclasses of datetime/date. - let ancestors = ob_type.getattr(intern!(py, "__mro__")).unwrap(); + let ancestors = ob_type.getattr(intern!(py, "__mro__"))?; let ancestors_str_iter = ancestors - .iter() - .unwrap() + .try_iter()? .map(|b| b.unwrap().str().unwrap().to_string()); for c in ancestors_str_iter { match &*c { diff --git a/crates/polars-python/src/conversion/chunked_array.rs b/crates/polars-python/src/conversion/chunked_array.rs index 404fe68ce8ef..de8ef187d4c4 100644 --- a/crates/polars-python/src/conversion/chunked_array.rs +++ b/crates/polars-python/src/conversion/chunked_array.rs @@ -1,90 +1,115 @@ use polars_core::export::chrono::NaiveTime; use polars_core::utils::arrow::temporal_conversions::date32_to_date; -use pyo3::intern; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyList, PyNone, PyTuple}; +use pyo3::{intern, BoundObject}; use super::datetime::{ elapsed_offset_to_timedelta, nanos_since_midnight_to_naivetime, timestamp_to_naive_datetime, }; use super::{decimal_to_digits, struct_dict}; use crate::prelude::*; -use crate::py_modules::UTILS; +use crate::py_modules::pl_utils; -impl ToPyObject for Wrap<&StringChunked> { - fn to_object(&self, py: Python) -> PyObject { +impl<'py> IntoPyObject<'py> for &Wrap<&StringChunked> { + type Target = PyList; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let iter = self.0.iter(); - PyList::new_bound(py, iter).into_py(py) + PyList::new(py, iter) } } -impl ToPyObject for Wrap<&BinaryChunked> { - fn to_object(&self, py: Python) -> PyObject { +impl<'py> IntoPyObject<'py> for &Wrap<&BinaryChunked> { + type Target = PyList; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let iter = self .0 .iter() - .map(|opt_bytes| opt_bytes.map(|bytes| PyBytes::new_bound(py, bytes))); - PyList::new_bound(py, iter).into_py(py) + .map(|opt_bytes| opt_bytes.map(|bytes| PyBytes::new(py, bytes))); + PyList::new(py, iter) } } -impl ToPyObject for Wrap<&StructChunked> { - fn to_object(&self, py: Python) -> PyObject { +impl<'py> IntoPyObject<'py> for &Wrap<&StructChunked> { + type Target = PyList; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + fn into_pyobject(self, py: Python<'py>) -> Result { let s = self.0.clone().into_series(); // todo! iterate its chunks and flatten. // make series::iter() accept a chunk index. let s = s.rechunk(); let iter = s.iter().map(|av| match av { - AnyValue::Struct(_, _, flds) => struct_dict(py, av._iter_struct_av(), flds), - AnyValue::Null => PyNone::get_bound(py).into_py(py), + AnyValue::Struct(_, _, flds) => struct_dict(py, av._iter_struct_av(), flds) + .unwrap() + .into_any(), + AnyValue::Null => PyNone::get(py).into_bound().into_any(), _ => unreachable!(), }); - PyList::new_bound(py, iter).into_py(py) + PyList::new(py, iter) } } -impl ToPyObject for Wrap<&DurationChunked> { - fn to_object(&self, py: Python) -> PyObject { +impl<'py> IntoPyObject<'py> for &Wrap<&DurationChunked> { + type Target = PyList; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let time_unit = self.0.time_unit(); let iter = self .0 .iter() .map(|opt_v| opt_v.map(|v| elapsed_offset_to_timedelta(v, time_unit))); - PyList::new_bound(py, iter).into_py(py) + PyList::new(py, iter) } } -impl ToPyObject for Wrap<&DatetimeChunked> { - fn to_object(&self, py: Python) -> PyObject { +impl<'py> IntoPyObject<'py> for &Wrap<&DatetimeChunked> { + type Target = PyList; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let time_zone = self.0.time_zone(); if time_zone.is_some() { // Switch to more efficient code path in // https://github.com/pola-rs/polars/issues/16199 - let utils = UTILS.bind(py); - let convert = utils.getattr(intern!(py, "to_py_datetime")).unwrap(); + let utils = pl_utils(py).bind(py); + let convert = utils.getattr(intern!(py, "to_py_datetime"))?; let time_unit = self.0.time_unit().to_ascii(); - let time_zone = time_zone.as_deref().to_object(py); + let time_zone = time_zone.as_deref().into_pyobject(py)?; let iter = self .0 .iter() .map(|opt_v| opt_v.map(|v| convert.call1((v, time_unit, &time_zone)).unwrap())); - PyList::new_bound(py, iter).into_py(py) + PyList::new(py, iter) } else { let time_unit = self.0.time_unit(); let iter = self .0 .iter() .map(|opt_v| opt_v.map(|v| timestamp_to_naive_datetime(v, time_unit))); - PyList::new_bound(py, iter).into_py(py) + PyList::new(py, iter) } } } -impl ToPyObject for Wrap<&TimeChunked> { - fn to_object(&self, py: Python) -> PyObject { +impl<'py> IntoPyObject<'py> for &Wrap<&TimeChunked> { + type Target = PyList; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let iter = time_to_pyobject_iter(self.0); - PyList::new_bound(py, iter).into_py(py) + PyList::new(py, iter) } } @@ -95,30 +120,37 @@ pub(crate) fn time_to_pyobject_iter( .map(move |opt_v| opt_v.map(nanos_since_midnight_to_naivetime)) } -impl ToPyObject for Wrap<&DateChunked> { - fn to_object(&self, py: Python) -> PyObject { +impl<'py> IntoPyObject<'py> for &Wrap<&DateChunked> { + type Target = PyList; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let iter = self.0.into_iter().map(|opt_v| opt_v.map(date32_to_date)); - PyList::new_bound(py, iter).into_py(py) + PyList::new(py, iter) } } -impl ToPyObject for Wrap<&DecimalChunked> { - fn to_object(&self, py: Python) -> PyObject { - let iter = decimal_to_pyobject_iter(py, self.0); - PyList::new_bound(py, iter).into_py(py) +impl<'py> IntoPyObject<'py> for &Wrap<&DecimalChunked> { + type Target = PyList; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + fn into_pyobject(self, py: Python<'py>) -> Result { + let iter = decimal_to_pyobject_iter(py, self.0)?; + PyList::new(py, iter) } } -pub(crate) fn decimal_to_pyobject_iter<'a>( - py: Python<'a>, +pub(crate) fn decimal_to_pyobject_iter<'py, 'a>( + py: Python<'py>, ca: &'a DecimalChunked, -) -> impl ExactSizeIterator>> { - let utils = UTILS.bind(py); - let convert = utils.getattr(intern!(py, "to_py_decimal")).unwrap(); - let py_scale = (-(ca.scale() as i32)).to_object(py); +) -> PyResult>> + use<'py, 'a>> { + let utils = pl_utils(py).bind(py); + let convert = utils.getattr(intern!(py, "to_py_decimal"))?; + let py_scale = (-(ca.scale() as i32)).into_pyobject(py)?; // if we don't know precision, the only safe bet is to set it to 39 - let py_precision = ca.precision().unwrap_or(39).to_object(py); - ca.iter().map(move |opt_v| { + let py_precision = ca.precision().unwrap_or(39).into_pyobject(py)?; + Ok(ca.iter().map(move |opt_v| { opt_v.map(|v| { // TODO! use AnyValue so that we have a single impl. const N: usize = 3; @@ -130,10 +162,10 @@ pub(crate) fn decimal_to_pyobject_iter<'a>( N * size_of::(), ) }; - let digits = PyTuple::new_bound(py, buf.iter().take(n_digits)); + let digits = PyTuple::new(py, buf.iter().take(n_digits)).unwrap(); convert .call1((v.is_negative() as u8, digits, &py_precision, &py_scale)) .unwrap() }) - }) + })) } diff --git a/crates/polars-python/src/conversion/mod.rs b/crates/polars-python/src/conversion/mod.rs index 5111169015e3..5227ec884fc7 100644 --- a/crates/polars-python/src/conversion/mod.rs +++ b/crates/polars-python/src/conversion/mod.rs @@ -1,6 +1,8 @@ pub(crate) mod any_value; pub(crate) mod chunked_array; mod datetime; + +use std::convert::Infallible; use std::fmt::{Display, Formatter}; use std::fs::File; use std::hash::{Hash, Hasher}; @@ -29,14 +31,14 @@ use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::intern; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; -use pyo3::types::{PyDict, PyList, PySequence}; +use pyo3::types::{PyDict, PyList, PySequence, PyString}; use crate::error::PyPolarsErr; use crate::file::{get_python_scan_source_input, PythonScanSourceInput}; #[cfg(feature = "object")] use crate::object::OBJECT_NAME; use crate::prelude::*; -use crate::py_modules::{POLARS, SERIES}; +use crate::py_modules::{pl_series, polars}; use crate::series::PySeries; use crate::{PyDataFrame, PyLazyFrame}; @@ -106,12 +108,10 @@ pub(crate) fn get_series(obj: &Bound<'_, PyAny>) -> PyResult { Ok(s.extract::()?.series) } -pub(crate) fn to_series(py: Python, s: PySeries) -> PyObject { - let series = SERIES.bind(py); - let constructor = series - .getattr(intern!(series.py(), "_from_pyseries")) - .unwrap(); - constructor.call1((s,)).unwrap().into_py(py) +pub(crate) fn to_series(py: Python, s: PySeries) -> PyResult> { + let series = pl_series(py).bind(py); + let constructor = series.getattr(intern!(py, "_from_pyseries"))?; + constructor.call1((s,)) } impl<'a> FromPyObject<'a> for Wrap { @@ -144,16 +144,16 @@ impl<'a> FromPyObject<'a> for Wrap { } } -fn struct_dict<'a>( - py: Python, +fn struct_dict<'py, 'a>( + py: Python<'py>, vals: impl Iterator>, flds: &[Field], -) -> PyObject { - let dict = PyDict::new_bound(py); - for (fld, val) in flds.iter().zip(vals) { - dict.set_item(fld.name().as_str(), Wrap(val)).unwrap() - } - dict.into_py(py) +) -> PyResult> { + let dict = PyDict::new(py); + flds.iter().zip(vals).try_for_each(|(fld, val)| { + dict.set_item(fld.name().as_str(), Wrap(val).into_pyobject(py)?) + })?; + Ok(dict) } // accept u128 array to ensure alignment is correct @@ -179,142 +179,140 @@ fn decimal_to_digits(v: i128, buf: &mut [u128; 3]) -> usize { len } -impl ToPyObject for Wrap { - fn to_object(&self, py: Python) -> PyObject { - let pl = POLARS.bind(py); +impl<'py> IntoPyObject<'py> for &Wrap { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let pl = polars(py).bind(py); match &self.0 { DataType::Int8 => { - let class = pl.getattr(intern!(py, "Int8")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Int8"))?; + class.call0() }, DataType::Int16 => { - let class = pl.getattr(intern!(py, "Int16")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Int16"))?; + class.call0() }, DataType::Int32 => { - let class = pl.getattr(intern!(py, "Int32")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Int32"))?; + class.call0() }, DataType::Int64 => { - let class = pl.getattr(intern!(py, "Int64")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Int64"))?; + class.call0() }, DataType::UInt8 => { - let class = pl.getattr(intern!(py, "UInt8")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "UInt8"))?; + class.call0() }, DataType::UInt16 => { - let class = pl.getattr(intern!(py, "UInt16")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "UInt16"))?; + class.call0() }, DataType::UInt32 => { - let class = pl.getattr(intern!(py, "UInt32")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "UInt32"))?; + class.call0() }, DataType::UInt64 => { - let class = pl.getattr(intern!(py, "UInt64")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "UInt64"))?; + class.call0() }, DataType::Int128 => { - let class = pl.getattr(intern!(py, "Int128")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Int128"))?; + class.call0() }, DataType::Float32 => { - let class = pl.getattr(intern!(py, "Float32")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Float32"))?; + class.call0() }, DataType::Float64 | DataType::Unknown(UnknownKind::Float) => { - let class = pl.getattr(intern!(py, "Float64")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Float64"))?; + class.call0() }, DataType::Decimal(precision, scale) => { - let class = pl.getattr(intern!(py, "Decimal")).unwrap(); + let class = pl.getattr(intern!(py, "Decimal"))?; let args = (*precision, *scale); - class.call1(args).unwrap().into() + class.call1(args) }, DataType::Boolean => { - let class = pl.getattr(intern!(py, "Boolean")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Boolean"))?; + class.call0() }, DataType::String | DataType::Unknown(UnknownKind::Str) => { - let class = pl.getattr(intern!(py, "String")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "String"))?; + class.call0() }, DataType::Binary => { - let class = pl.getattr(intern!(py, "Binary")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Binary"))?; + class.call0() }, DataType::Array(inner, size) => { - let class = pl.getattr(intern!(py, "Array")).unwrap(); - let inner = Wrap(*inner.clone()).to_object(py); - let args = (inner, *size); - class.call1(args).unwrap().into() + let class = pl.getattr(intern!(py, "Array"))?; + let inner = Wrap(*inner.clone()); + let args = (&inner, *size); + class.call1(args) }, DataType::List(inner) => { - let class = pl.getattr(intern!(py, "List")).unwrap(); - let inner = Wrap(*inner.clone()).to_object(py); - class.call1((inner,)).unwrap().into() + let class = pl.getattr(intern!(py, "List"))?; + let inner = Wrap(*inner.clone()); + class.call1((&inner,)) }, DataType::Date => { - let class = pl.getattr(intern!(py, "Date")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Date"))?; + class.call0() }, DataType::Datetime(tu, tz) => { - let datetime_class = pl.getattr(intern!(py, "Datetime")).unwrap(); - datetime_class - .call1((tu.to_ascii(), tz.as_deref())) - .unwrap() - .into() + let datetime_class = pl.getattr(intern!(py, "Datetime"))?; + datetime_class.call1((tu.to_ascii(), tz.as_deref())) }, DataType::Duration(tu) => { - let duration_class = pl.getattr(intern!(py, "Duration")).unwrap(); - duration_class.call1((tu.to_ascii(),)).unwrap().into() + let duration_class = pl.getattr(intern!(py, "Duration"))?; + duration_class.call1((tu.to_ascii(),)) }, #[cfg(feature = "object")] DataType::Object(_, _) => { - let class = pl.getattr(intern!(py, "Object")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Object"))?; + class.call0() }, DataType::Categorical(_, ordering) => { - let class = pl.getattr(intern!(py, "Categorical")).unwrap(); - class - .call1((Wrap(*ordering).to_object(py),)) - .unwrap() - .into() + let class = pl.getattr(intern!(py, "Categorical"))?; + class.call1((Wrap(*ordering),)) }, DataType::Enum(rev_map, _) => { // we should always have an initialized rev_map coming from rust let categories = rev_map.as_ref().unwrap().get_categories(); - let class = pl.getattr(intern!(py, "Enum")).unwrap(); + let class = pl.getattr(intern!(py, "Enum"))?; let s = Series::from_arrow(PlSmallStr::from_static("category"), categories.to_boxed()) - .unwrap(); - let series = to_series(py, s.into()); - class.call1((series,)).unwrap().into() + .map_err(PyPolarsErr::from)?; + let series = to_series(py, s.into())?; + class.call1((series,)) }, - DataType::Time => pl.getattr(intern!(py, "Time")).unwrap().into(), + DataType::Time => pl.getattr(intern!(py, "Time")), DataType::Struct(fields) => { - let field_class = pl.getattr(intern!(py, "Field")).unwrap(); + let field_class = pl.getattr(intern!(py, "Field"))?; let iter = fields.iter().map(|fld| { let name = fld.name().as_str(); - let dtype = Wrap(fld.dtype().clone()).to_object(py); - field_class.call1((name, dtype)).unwrap() + let dtype = Wrap(fld.dtype().clone()); + field_class.call1((name, &dtype)).unwrap() }); - let fields = PyList::new_bound(py, iter); - let struct_class = pl.getattr(intern!(py, "Struct")).unwrap(); - struct_class.call1((fields,)).unwrap().into() + let fields = PyList::new(py, iter)?; + let struct_class = pl.getattr(intern!(py, "Struct"))?; + struct_class.call1((fields,)) }, DataType::Null => { - let class = pl.getattr(intern!(py, "Null")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Null"))?; + class.call0() }, DataType::Unknown(UnknownKind::Int(v)) => { - Wrap(materialize_dyn_int(*v).dtype()).to_object(py) + Wrap(materialize_dyn_int(*v).dtype()).into_pyobject(py) }, DataType::Unknown(_) => { - let class = pl.getattr(intern!(py, "Unknown")).unwrap(); - class.call0().unwrap().into() + let class = pl.getattr(intern!(py, "Unknown"))?; + class.call0() }, DataType::BinaryOffset => { unimplemented!() @@ -465,24 +463,27 @@ impl<'py> FromPyObject<'py> for Wrap { } } -impl ToPyObject for Wrap { - fn to_object(&self, py: Python<'_>) -> PyObject { - let ordering = match self.0 { +impl<'py> IntoPyObject<'py> for Wrap { + type Target = PyString; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + match self.0 { CategoricalOrdering::Physical => "physical", CategoricalOrdering::Lexical => "lexical", - }; - ordering.into_py(py) + } + .into_pyobject(py) } } -impl ToPyObject for Wrap { - fn to_object(&self, py: Python) -> PyObject { - let time_unit = match self.0 { - TimeUnit::Nanoseconds => "ns", - TimeUnit::Microseconds => "us", - TimeUnit::Milliseconds => "ms", - }; - time_unit.into_py(py) +impl<'py> IntoPyObject<'py> for Wrap { + type Target = PyString; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + self.0.to_ascii().into_pyobject(py) } } @@ -600,13 +601,17 @@ impl<'py> FromPyObject<'py> for Wrap { } } -impl IntoPy for Wrap<&Schema> { - fn into_py(self, py: Python<'_>) -> PyObject { - let dict = PyDict::new_bound(py); - for (k, v) in self.0.iter() { - dict.set_item(k.as_str(), Wrap(v.clone())).unwrap(); - } - dict.into_py(py) +impl<'py> IntoPyObject<'py> for Wrap<&Schema> { + type Target = PyDict; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let dict = PyDict::new(py); + self.0 + .iter() + .try_for_each(|(k, v)| dict.set_item(k.as_str(), &Wrap(v.clone())))?; + Ok(dict) } } @@ -684,10 +689,8 @@ impl From for ObjectValue { impl<'a> FromPyObject<'a> for ObjectValue { fn extract_bound(ob: &Bound<'a, PyAny>) -> PyResult { - Python::with_gil(|py| { - Ok(ObjectValue { - inner: ob.to_object(py), - }) + Ok(ObjectValue { + inner: ob.to_owned().unbind(), }) } } @@ -702,9 +705,13 @@ impl From<&dyn PolarsObjectSafe> for &ObjectValue { } } -impl ToPyObject for ObjectValue { - fn to_object(&self, py: Python) -> PyObject { - self.inner.clone_ref(py) +impl<'a, 'py> IntoPyObject<'py> for &'a ObjectValue { + type Target = PyAny; + type Output = Borrowed<'a, 'py, Self::Target>; + type Error = std::convert::Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + Ok(self.inner.bind_borrowed(py)) } } @@ -718,7 +725,7 @@ impl<'a, T: NativeType + FromPyObject<'a>> FromPyObject<'a> for Wrap> { fn extract_bound(obj: &Bound<'a, PyAny>) -> PyResult { let seq = obj.downcast::()?; let mut v = Vec::with_capacity(seq.len().unwrap_or(0)); - for item in seq.iter()? { + for item in seq.try_iter()? { v.push(item?.extract::()?); } Ok(Wrap(v)) diff --git a/crates/polars-python/src/dataframe/construction.rs b/crates/polars-python/src/dataframe/construction.rs index 8520caf220d9..df051cab1741 100644 --- a/crates/polars-python/src/dataframe/construction.rs +++ b/crates/polars-python/src/dataframe/construction.rs @@ -145,7 +145,7 @@ fn dicts_to_rows<'a>( ) -> PyResult>> { let len = data.len()?; let mut rows = Vec::with_capacity(len); - for d in data.iter()? { + for d in data.try_iter()? { let d = d?; let d = d.downcast::()?; @@ -189,7 +189,7 @@ fn infer_schema_names_from_data( .unwrap_or(data_len); let mut names = PlIndexSet::new(); - for d in data.iter()?.take(infer_schema_length) { + for d in data.try_iter()?.take(infer_schema_length) { let d = d?; let d = d.downcast::()?; let keys = d.keys(); diff --git a/crates/polars-python/src/dataframe/export.rs b/crates/polars-python/src/dataframe/export.rs index 7486d9742467..5edb1ed5ed0a 100644 --- a/crates/polars-python/src/dataframe/export.rs +++ b/crates/polars-python/src/dataframe/export.rs @@ -4,6 +4,7 @@ use polars_core::export::arrow::datatypes::IntegerType; use polars_core::export::cast::CastOptionsImpl; use pyo3::prelude::*; use pyo3::types::{PyCapsule, PyList, PyTuple}; +use pyo3::IntoPyObjectExt; use super::PyDataFrame; use crate::conversion::{ObjectValue, Wrap}; @@ -15,7 +16,7 @@ use crate::prelude::PyCompatLevel; #[pymethods] impl PyDataFrame { #[cfg(feature = "object")] - pub fn row_tuple(&self, idx: i64) -> PyResult { + pub fn row_tuple<'py>(&self, idx: i64, py: Python<'py>) -> PyResult> { let idx = if idx < 0 { (self.df.height() as i64 + idx) as usize } else { @@ -24,70 +25,63 @@ impl PyDataFrame { if idx >= self.df.height() { return Err(PyPolarsErr::from(polars_err!(oob = idx, self.df.height())).into()); } - let out = Python::with_gil(|py| { - PyTuple::new_bound( - py, - self.df.get_columns().iter().map(|s| match s.dtype() { - DataType::Object(_, _) => { - let obj: Option<&ObjectValue> = s.get_object(idx).map(|any| any.into()); - obj.to_object(py) - }, - _ => Wrap(s.get(idx).unwrap()).into_py(py), - }), - ) - .into_py(py) - }); - Ok(out) + PyTuple::new( + py, + self.df.get_columns().iter().map(|s| match s.dtype() { + DataType::Object(_, _) => { + let obj: Option<&ObjectValue> = s.get_object(idx).map(|any| any.into()); + obj.into_py_any(py).unwrap() + }, + _ => Wrap(s.get(idx).unwrap()).into_py_any(py).unwrap(), + }), + ) } #[cfg(feature = "object")] - pub fn row_tuples(&self) -> PyObject { - Python::with_gil(|py| { - let mut rechunked; - // Rechunk if random access would become rather expensive. - // TODO: iterate over the chunks directly instead of using random access. - let df = if self.df.max_n_chunks() > 16 { - rechunked = self.df.clone(); - rechunked.as_single_chunk_par(); - &rechunked - } else { - &self.df - }; - PyList::new_bound( - py, - (0..df.height()).map(|idx| { - PyTuple::new_bound( - py, - df.get_columns().iter().map(|c| match c.dtype() { - DataType::Null => py.None(), - DataType::Object(_, _) => { - let obj: Option<&ObjectValue> = - c.get_object(idx).map(|any| any.into()); - obj.to_object(py) - }, - _ => { - // SAFETY: we are in bounds. - let av = unsafe { c.get_unchecked(idx) }; - Wrap(av).into_py(py) - }, - }), - ) - }), - ) - .into_py(py) - }) + pub fn row_tuples<'py>(&self, py: Python<'py>) -> PyResult> { + let mut rechunked; + // Rechunk if random access would become rather expensive. + // TODO: iterate over the chunks directly instead of using random access. + let df = if self.df.max_n_chunks() > 16 { + rechunked = self.df.clone(); + rechunked.as_single_chunk_par(); + &rechunked + } else { + &self.df + }; + PyList::new( + py, + (0..df.height()).map(|idx| { + PyTuple::new( + py, + df.get_columns().iter().map(|c| match c.dtype() { + DataType::Null => py.None(), + DataType::Object(_, _) => { + let obj: Option<&ObjectValue> = c.get_object(idx).map(|any| any.into()); + obj.into_py_any(py).unwrap() + }, + _ => { + // SAFETY: we are in bounds. + let av = unsafe { c.get_unchecked(idx) }; + Wrap(av).into_py_any(py).unwrap() + }, + }), + ) + .unwrap() + }), + ) } #[allow(clippy::wrong_self_convention)] pub fn to_arrow(&mut self, py: Python, compat_level: PyCompatLevel) -> PyResult> { py.allow_threads(|| self.df.align_chunks_par()); - let pyarrow = py.import_bound("pyarrow")?; + let pyarrow = py.import("pyarrow")?; let names = self.df.get_column_names_str(); let rbs = self .df .iter_chunks(compat_level.0, true) - .map(|rb| interop::arrow::to_py::to_py_rb(&rb, &names, py, &pyarrow)) + .map(|rb| interop::arrow::to_py::to_py_rb(&rb, &names, &pyarrow)) .collect::>()?; Ok(rbs) } @@ -101,7 +95,7 @@ impl PyDataFrame { pub fn to_pandas(&mut self, py: Python) -> PyResult> { py.allow_threads(|| self.df.as_single_chunk_par()); Python::with_gil(|py| { - let pyarrow = py.import_bound("pyarrow")?; + let pyarrow = py.import("pyarrow")?; let names = self.df.get_column_names_str(); let cat_columns = self .df @@ -138,7 +132,7 @@ impl PyDataFrame { } let rb = RecordBatch::new(length, rb); - interop::arrow::to_py::to_py_rb(&rb, &names, py, &pyarrow) + interop::arrow::to_py::to_py_rb(&rb, &names, &pyarrow) }) .collect::>()?; Ok(rbs) diff --git a/crates/polars-python/src/dataframe/general.rs b/crates/polars-python/src/dataframe/general.rs index 80741dea3700..00f7534086a2 100644 --- a/crates/polars-python/src/dataframe/general.rs +++ b/crates/polars-python/src/dataframe/general.rs @@ -10,6 +10,7 @@ use pyo3::exceptions::PyIndexError; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; use pyo3::types::PyList; +use pyo3::IntoPyObjectExt; use self::row_encode::get_row_encoding_dictionary; use super::PyDataFrame; @@ -20,6 +21,7 @@ use crate::map::dataframe::{ apply_lambda_with_string_out_type, }; use crate::prelude::strings_to_pl_smallstr; +use crate::py_modules::polars; use crate::series::{PySeries, ToPySeries, ToSeries}; use crate::{PyExpr, PyLazyFrame}; @@ -179,12 +181,12 @@ impl PyDataFrame { } /// Get datatypes - pub fn dtypes(&self, py: Python) -> PyObject { + pub fn dtypes<'py>(&self, py: Python<'py>) -> PyResult> { let iter = self .df .iter() - .map(|s| Wrap(s.dtype().clone()).to_object(py)); - PyList::new_bound(py, iter).to_object(py) + .map(|s| Wrap(s.dtype().clone()).into_pyobject(py).unwrap()); + PyList::new(py, iter) } pub fn n_chunks(&self) -> usize { @@ -401,7 +403,7 @@ impl PyDataFrame { let function = move |df: DataFrame| { Python::with_gil(|py| { - let pypolars = PyModule::import_bound(py, "polars").unwrap(); + let pypolars = polars(py).bind(py); let pydf = PyDataFrame::new(df); let python_df_wrapper = pypolars.getattr("wrap_df").unwrap().call1((pydf,)).unwrap(); @@ -409,7 +411,7 @@ impl PyDataFrame { // Call the lambda and get a python-side DataFrame wrapper. let result_df_wrapper = match lambda.call1(py, (python_df_wrapper,)) { Ok(pyobj) => pyobj, - Err(e) => panic!("UDF failed: {}", e.value_bound(py)), + Err(e) => panic!("UDF failed: {}", e.value(py)), }; let py_pydf = result_df_wrapper.getattr(py, "_df").expect( "Could not get DataFrame attribute '_df'. Make sure that you return a DataFrame object.", @@ -567,7 +569,7 @@ impl PyDataFrame { _ => return apply_lambda_unknown(df, py, lambda, inference_size), }; - Ok((PySeries::from(out).into_py(py), false)) + Ok((PySeries::from(out).into_py_any(py)?, false)) }) } diff --git a/crates/polars-python/src/dataframe/serde.rs b/crates/polars-python/src/dataframe/serde.rs index ac534cfee35d..c421dee342b7 100644 --- a/crates/polars-python/src/dataframe/serde.rs +++ b/crates/polars-python/src/dataframe/serde.rs @@ -15,14 +15,14 @@ use crate::file::{get_file_like, get_mmap_bytes_reader}; #[pymethods] impl PyDataFrame { #[cfg(feature = "ipc_streaming")] - fn __getstate__(&self, py: Python) -> PyResult { + fn __getstate__<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> { // Used in pickle/pickling let mut buf: Vec = vec![]; IpcStreamWriter::new(&mut buf) .with_compat_level(CompatLevel::newest()) .finish(&mut self.df.clone()) .expect("ipc writer"); - Ok(PyBytes::new_bound(py, &buf).to_object(py)) + PyBytes::new(py, &buf) } #[cfg(feature = "ipc_streaming")] diff --git a/crates/polars-python/src/error.rs b/crates/polars-python/src/error.rs index 1a84ad5517fb..dc04883ebeb1 100644 --- a/crates/polars-python/src/error.rs +++ b/crates/polars-python/src/error.rs @@ -102,16 +102,20 @@ macro_rules! raise_err( }} ); -impl IntoPy for Wrap { - fn into_py(self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for Wrap { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = std::convert::Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { match self.0 { PolarsWarning::CategoricalRemappingWarning => { - CategoricalRemappingWarning::type_object_bound(py).to_object(py) + Ok(CategoricalRemappingWarning::type_object(py).into_any()) }, PolarsWarning::MapWithoutReturnDtypeWarning => { - MapWithoutReturnDtypeWarning::type_object_bound(py).to_object(py) + Ok(MapWithoutReturnDtypeWarning::type_object(py).into_any()) }, - PolarsWarning::UserWarning => PyUserWarning::type_object_bound(py).to_object(py), + PolarsWarning::UserWarning => Ok(PyUserWarning::type_object(py).into_any()), } } } diff --git a/crates/polars-python/src/exceptions.rs b/crates/polars-python/src/exceptions.rs index 1e020d19fec5..3edbd6c71de2 100644 --- a/crates/polars-python/src/exceptions.rs +++ b/crates/polars-python/src/exceptions.rs @@ -1,9 +1,5 @@ //! Define the Polars exception hierarchy. -// TODO: Remove this directive when upgrading to PyO3 version 0.23. -// https://github.com/PyO3/pyo3/issues/4743 -#![allow(unexpected_cfgs)] - use pyo3::create_exception; use pyo3::exceptions::{PyException, PyWarning}; diff --git a/crates/polars-python/src/expr/list.rs b/crates/polars-python/src/expr/list.rs index af3be10449b1..b8f10fc60c3e 100644 --- a/crates/polars-python/src/expr/list.rs +++ b/crates/polars-python/src/expr/list.rs @@ -244,7 +244,7 @@ impl PyExpr { .list() .to_struct(ListToStructArgs::FixedWidth( names - .iter()? + .try_iter()? .map(|x| Ok(x?.extract::>()?.0)) .collect::>>()?, )) diff --git a/crates/polars-python/src/expr/serde.rs b/crates/polars-python/src/expr/serde.rs index 9933e9a979b1..0900c40e0722 100644 --- a/crates/polars-python/src/expr/serde.rs +++ b/crates/polars-python/src/expr/serde.rs @@ -12,26 +12,22 @@ use crate::PyExpr; #[pymethods] impl PyExpr { - fn __getstate__(&self, py: Python) -> PyResult { + fn __getstate__<'py>(&self, py: Python<'py>) -> PyResult> { // Used in pickle/pickling let mut writer: Vec = vec![]; ciborium::ser::into_writer(&self.inner, &mut writer) .map_err(|e| PyPolarsErr::Other(format!("{}", e)))?; - Ok(PyBytes::new_bound(py, &writer).to_object(py)) + Ok(PyBytes::new(py, &writer)) } fn __setstate__(&mut self, state: &Bound) -> PyResult<()> { // Used in pickle/pickling - match state.extract::() { - Ok(s) => { - let cursor = Cursor::new(&*s); - self.inner = ciborium::de::from_reader(cursor) - .map_err(|e| PyPolarsErr::Other(format!("{}", e)))?; - Ok(()) - }, - Err(e) => Err(e), - } + let bytes = state.extract::()?; + let cursor = Cursor::new(&*bytes); + self.inner = + ciborium::de::from_reader(cursor).map_err(|e| PyPolarsErr::Other(format!("{}", e)))?; + Ok(()) } /// Serialize into binary data. diff --git a/crates/polars-python/src/file.rs b/crates/polars-python/src/file.rs index 996a63ea1e48..e417c59c9f92 100644 --- a/crates/polars-python/src/file.rs +++ b/crates/polars-python/src/file.rs @@ -16,6 +16,7 @@ use polars_utils::mmap::MemSlice; use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyString, PyStringMethods}; +use pyo3::IntoPyObjectExt; use crate::error::PyPolarsErr; use crate::prelude::resolve_homedir; @@ -45,7 +46,7 @@ impl PyFileLikeObject { Python::with_gil(|py| { let bytes = self .inner - .call_method_bound(py, "read", (), None) + .call_method(py, "read", (), None) .expect("no read method found"); if let Ok(b) = bytes.downcast_bound::(py) { @@ -99,9 +100,9 @@ impl PyFileLikeObject { /// Extracts a string repr from, and returns an IO error to send back to rust. fn pyerr_to_io_err(e: PyErr) -> io::Error { Python::with_gil(|py| { - let e_as_object: PyObject = e.into_py(py); + let e_as_object: PyObject = e.into_py_any(py).unwrap(); - match e_as_object.call_method_bound(py, "__str__", (), None) { + match e_as_object.call_method(py, "__str__", (), None) { Ok(repr) => match repr.extract::(py) { Ok(s) => io::Error::new(io::ErrorKind::Other, s), Err(_e) => io::Error::new(io::ErrorKind::Other, "An unknown error has occurred"), @@ -116,7 +117,7 @@ impl Read for PyFileLikeObject { Python::with_gil(|py| { let bytes = self .inner - .call_method_bound(py, "read", (buf.len(),), None) + .call_method(py, "read", (buf.len(),), None) .map_err(pyerr_to_io_err)?; let opt_bytes = bytes.downcast_bound::(py); @@ -142,11 +143,11 @@ impl Read for PyFileLikeObject { impl Write for PyFileLikeObject { fn write(&mut self, buf: &[u8]) -> Result { Python::with_gil(|py| { - let pybytes = PyBytes::new_bound(py, buf); + let pybytes = PyBytes::new(py, buf); let number_bytes_written = self .inner - .call_method_bound(py, "write", (pybytes,), None) + .call_method(py, "write", (pybytes,), None) .map_err(pyerr_to_io_err)?; number_bytes_written.extract(py).map_err(pyerr_to_io_err) @@ -156,7 +157,7 @@ impl Write for PyFileLikeObject { fn flush(&mut self) -> Result<(), io::Error> { Python::with_gil(|py| { self.inner - .call_method_bound(py, "flush", (), None) + .call_method(py, "flush", (), None) .map_err(pyerr_to_io_err)?; Ok(()) @@ -175,7 +176,7 @@ impl Seek for PyFileLikeObject { let new_position = self .inner - .call_method_bound(py, "seek", (offset, whence), None) + .call_method(py, "seek", (offset, whence), None) .map_err(pyerr_to_io_err)?; new_position.extract(py).map_err(pyerr_to_io_err) @@ -228,7 +229,7 @@ fn try_get_pyfile( py_f: Bound<'_, PyAny>, write: bool, ) -> PyResult<(EitherRustPythonFile, Option)> { - let io = py.import_bound("io").unwrap(); + let io = py.import("io")?; let is_utf8_encoding = |py_f: &Bound| -> PyResult { let encoding = py_f.getattr("encoding")?; let encoding = encoding.extract::>()?; @@ -311,7 +312,7 @@ fn try_get_pyfile( py_f }; PyFileLikeObject::ensure_requirements(&py_f, !write, write, !write)?; - let f = PyFileLikeObject::new(py_f.to_object(py)); + let f = PyFileLikeObject::new(py_f.unbind()); Ok((EitherRustPythonFile::Py(f), None)) } @@ -400,14 +401,14 @@ pub fn get_mmap_bytes_reader_and_path( Ok(( Box::new(Cursor::new(MemSlice::from_arc( bytes.as_bytes(), - Arc::new(py_f.to_object(py_f.py())), + Arc::new(py_f.clone().unbind()), ))), None, )) } // string so read file else { - match get_either_buffer_or_path(py_f.to_object(py_f.py()), false)? { + match get_either_buffer_or_path(py_f.to_owned().unbind(), false)? { (EitherRustPythonFile::Rust(f), path) => Ok((Box::new(f), path)), (EitherRustPythonFile::Py(f), path) => { Ok((Box::new(Cursor::new(f.to_memslice())), path)) diff --git a/crates/polars-python/src/functions/eager.rs b/crates/polars-python/src/functions/eager.rs index f271b818c8ce..5f68067824f1 100644 --- a/crates/polars-python/src/functions/eager.rs +++ b/crates/polars-python/src/functions/eager.rs @@ -11,7 +11,7 @@ pub fn concat_df(dfs: &Bound<'_, PyAny>, py: Python) -> PyResult { use polars_core::error::PolarsResult; use polars_core::utils::rayon::prelude::*; - let mut iter = dfs.iter()?; + let mut iter = dfs.try_iter()?; let first = iter.next().unwrap()?; let first_rdf = get_df(&first)?; @@ -49,7 +49,7 @@ pub fn concat_df(dfs: &Bound<'_, PyAny>, py: Python) -> PyResult { #[pyfunction] pub fn concat_series(series: &Bound<'_, PyAny>) -> PyResult { - let mut iter = series.iter()?; + let mut iter = series.try_iter()?; let first = iter.next().unwrap()?; let mut s = get_series(&first)?; @@ -64,7 +64,7 @@ pub fn concat_series(series: &Bound<'_, PyAny>) -> PyResult { #[pyfunction] pub fn concat_df_diagonal(dfs: &Bound<'_, PyAny>) -> PyResult { - let iter = dfs.iter()?; + let iter = dfs.try_iter()?; let dfs = iter .map(|item| { @@ -79,7 +79,7 @@ pub fn concat_df_diagonal(dfs: &Bound<'_, PyAny>) -> PyResult { #[pyfunction] pub fn concat_df_horizontal(dfs: &Bound<'_, PyAny>) -> PyResult { - let iter = dfs.iter()?; + let iter = dfs.try_iter()?; let dfs = iter .map(|item| { diff --git a/crates/polars-python/src/functions/io.rs b/crates/polars-python/src/functions/io.rs index 80b0c44b3f3c..064b31793ede 100644 --- a/crates/polars-python/src/functions/io.rs +++ b/crates/polars-python/src/functions/io.rs @@ -14,7 +14,7 @@ use crate::prelude::ArrowDataType; #[cfg(feature = "ipc")] #[pyfunction] -pub fn read_ipc_schema(py: Python, py_f: PyObject) -> PyResult { +pub fn read_ipc_schema(py: Python, py_f: PyObject) -> PyResult> { use polars_core::export::arrow::io::ipc::read::read_file_metadata; let metadata = match get_either_file(py_f, false)? { EitherRustPythonFile::Rust(r) => { @@ -23,14 +23,14 @@ pub fn read_ipc_schema(py: Python, py_f: PyObject) -> PyResult { EitherRustPythonFile::Py(mut r) => read_file_metadata(&mut r).map_err(PyPolarsErr::from)?, }; - let dict = PyDict::new_bound(py); - fields_to_pydict(&metadata.schema, &dict, py)?; - Ok(dict.to_object(py)) + let dict = PyDict::new(py); + fields_to_pydict(&metadata.schema, &dict)?; + Ok(dict) } #[cfg(feature = "parquet")] #[pyfunction] -pub fn read_parquet_schema(py: Python, py_f: PyObject) -> PyResult { +pub fn read_parquet_schema(py: Python, py_f: PyObject) -> PyResult> { use polars_parquet::read::{infer_schema, read_metadata}; let metadata = match get_either_file(py_f, false)? { @@ -41,13 +41,13 @@ pub fn read_parquet_schema(py: Python, py_f: PyObject) -> PyResult { }; let arrow_schema = infer_schema(&metadata).map_err(PyPolarsErr::from)?; - let dict = PyDict::new_bound(py); - fields_to_pydict(&arrow_schema, &dict, py)?; - Ok(dict.to_object(py)) + let dict = PyDict::new(py); + fields_to_pydict(&arrow_schema, &dict)?; + Ok(dict) } #[cfg(any(feature = "ipc", feature = "parquet"))] -fn fields_to_pydict(schema: &ArrowSchema, dict: &Bound<'_, PyDict>, py: Python) -> PyResult<()> { +fn fields_to_pydict(schema: &ArrowSchema, dict: &Bound<'_, PyDict>) -> PyResult<()> { for field in schema.iter_values() { let dt = if field.is_enum() { Wrap(create_enum_dtype(Utf8ViewArray::new_empty( @@ -56,7 +56,7 @@ fn fields_to_pydict(schema: &ArrowSchema, dict: &Bound<'_, PyDict>, py: Python) } else { Wrap(polars::prelude::DataType::from_arrow_field(field)) }; - dict.set_item(field.name.as_str(), dt.to_object(py))?; + dict.set_item(field.name.as_str(), &dt)?; } Ok(()) } diff --git a/crates/polars-python/src/functions/lazy.rs b/crates/polars-python/src/functions/lazy.rs index 5bc8819f3f85..e4aef88ef6c8 100644 --- a/crates/polars-python/src/functions/lazy.rs +++ b/crates/polars-python/src/functions/lazy.rs @@ -10,7 +10,7 @@ use crate::conversion::{get_lf, Wrap}; use crate::error::PyPolarsErr; use crate::expr::ToExprs; use crate::map::lazy::binary_lambda; -use crate::prelude::{vec_extract_wrapped, ObjectValue}; +use crate::prelude::vec_extract_wrapped; use crate::{map, PyDataFrame, PyExpr, PyLazyFrame, PySeries}; macro_rules! set_unwrapped_or_0 { @@ -151,7 +151,7 @@ pub fn collect_all_with_callback(lfs: Vec, lambda: PyObject) { }, Err(err) => { lambda - .call1(py, (PyErr::from(err).to_object(py),)) + .call1(py, (PyErr::from(err),)) .map_err(|err| err.restore(py)) .ok(); }, @@ -174,7 +174,7 @@ pub fn concat_lf( let len = seq.len()?; let mut lfs = Vec::with_capacity(len); - for res in seq.iter()? { + for res in seq.try_iter()? { let item = res?; let lf = get_lf(&item)?; lfs.push(lf); @@ -302,7 +302,7 @@ pub fn concat_lf_diagonal( parallel: bool, to_supertypes: bool, ) -> PyResult { - let iter = lfs.iter()?; + let iter = lfs.try_iter()?; let lfs = iter .map(|item| { @@ -326,7 +326,7 @@ pub fn concat_lf_diagonal( #[pyfunction] pub fn concat_lf_horizontal(lfs: &Bound<'_, PyAny>, parallel: bool) -> PyResult { - let iter = lfs.iter()?; + let iter = lfs.try_iter()?; let lfs = iter .map(|item| { @@ -437,8 +437,9 @@ pub fn nth(n: i64) -> PyExpr { #[pyfunction] pub fn lit(value: &Bound<'_, PyAny>, allow_object: bool, is_scalar: bool) -> PyResult { + let py = value.py(); if value.is_instance_of::() { - let val = value.extract::().unwrap(); + let val = value.extract::()?; Ok(dsl::lit(val).into()) } else if let Ok(int) = value.downcast::() { let v = int @@ -447,7 +448,7 @@ pub fn lit(value: &Bound<'_, PyAny>, allow_object: bool, is_scalar: bool) -> PyR .map_err(PyPolarsErr::from)?; Ok(Expr::Literal(LiteralValue::Int(v)).into()) } else if let Ok(float) = value.downcast::() { - let val = float.extract::().unwrap(); + let val = float.extract::()?; Ok(Expr::Literal(LiteralValue::Float(val)).into()) } else if let Ok(pystr) = value.downcast::() { Ok(dsl::lit(pystr.to_string()).into()) @@ -479,10 +480,7 @@ pub fn lit(value: &Bound<'_, PyAny>, allow_object: bool, is_scalar: bool) -> PyR match av { #[cfg(feature = "object")] AnyValue::ObjectOwned(_) => { - let s = Python::with_gil(|py| { - PySeries::new_object(py, "", vec![ObjectValue::from(value.into_py(py))], false) - .series - }); + let s = PySeries::new_object(py, "", vec![value.extract()?], false).series; Ok(dsl::lit(s).into()) }, _ => Ok(Expr::Literal(LiteralValue::from(av)).into()), diff --git a/crates/polars-python/src/functions/meta.rs b/crates/polars-python/src/functions/meta.rs index ba6af7f2c669..8502c30bdb8d 100644 --- a/crates/polars-python/src/functions/meta.rs +++ b/crates/polars-python/src/functions/meta.rs @@ -7,8 +7,8 @@ use pyo3::prelude::*; use crate::conversion::Wrap; #[pyfunction] -pub fn get_index_type(py: Python) -> PyObject { - Wrap(IDX_DTYPE).to_object(py) +pub fn get_index_type(py: Python) -> PyResult> { + Wrap(IDX_DTYPE).into_pyobject(py) } #[pyfunction] diff --git a/crates/polars-python/src/interop/arrow/to_py.rs b/crates/polars-python/src/interop/arrow/to_py.rs index e3a77e80837b..abbd763ed36e 100644 --- a/crates/polars-python/src/interop/arrow/to_py.rs +++ b/crates/polars-python/src/interop/arrow/to_py.rs @@ -14,11 +14,7 @@ use pyo3::prelude::*; use pyo3::types::PyCapsule; /// Arrow array to Python. -pub(crate) fn to_py_array( - array: ArrayRef, - py: Python, - pyarrow: &Bound, -) -> PyResult { +pub(crate) fn to_py_array(array: ArrayRef, pyarrow: &Bound) -> PyResult { let schema = Box::new(ffi::export_field_to_c(&ArrowField::new( PlSmallStr::EMPTY, array.dtype().clone(), @@ -34,20 +30,19 @@ pub(crate) fn to_py_array( (array_ptr as Py_uintptr_t, schema_ptr as Py_uintptr_t), )?; - Ok(array.to_object(py)) + Ok(array.unbind()) } /// RecordBatch to Python. pub(crate) fn to_py_rb( rb: &RecordBatch, names: &[&str], - py: Python, pyarrow: &Bound, ) -> PyResult { let mut arrays = Vec::with_capacity(rb.len()); for array in rb.columns() { - let array_object = to_py_array(array.clone(), py, pyarrow)?; + let array_object = to_py_array(array.clone(), pyarrow)?; arrays.push(array_object); } @@ -55,7 +50,7 @@ pub(crate) fn to_py_rb( .getattr("RecordBatch")? .call_method1("from_arrays", (arrays, names.to_vec()))?; - Ok(record.to_object(py)) + Ok(record.unbind()) } /// Export a series to a C stream via a PyCapsule according to the Arrow PyCapsule Interface @@ -68,7 +63,7 @@ pub(crate) fn series_to_stream<'py>( let iter = Box::new(series.chunks().clone().into_iter().map(Ok)) as _; let stream = ffi::export_iterator(iter, field); let stream_capsule_name = CString::new("arrow_array_stream").unwrap(); - PyCapsule::new_bound(py, stream, Some(stream_capsule_name)) + PyCapsule::new(py, stream, Some(stream_capsule_name)) } pub(crate) fn dataframe_to_stream<'py>( @@ -79,7 +74,7 @@ pub(crate) fn dataframe_to_stream<'py>( let field = iter.field(); let stream = ffi::export_iterator(iter, field); let stream_capsule_name = CString::new("arrow_array_stream").unwrap(); - PyCapsule::new_bound(py, stream, Some(stream_capsule_name)) + PyCapsule::new(py, stream, Some(stream_capsule_name)) } pub struct DataFrameStreamIterator { diff --git a/crates/polars-python/src/interop/numpy/to_numpy_df.rs b/crates/polars-python/src/interop/numpy/to_numpy_df.rs index 96374ab4236a..5e0b61774dd6 100644 --- a/crates/polars-python/src/interop/numpy/to_numpy_df.rs +++ b/crates/polars-python/src/interop/numpy/to_numpy_df.rs @@ -5,9 +5,9 @@ use polars_core::prelude::*; use polars_core::utils::dtypes_to_supertype; use polars_core::with_match_physical_numeric_polars_type; use pyo3::exceptions::PyRuntimeError; -use pyo3::intern; use pyo3::prelude::*; use pyo3::types::PyList; +use pyo3::{intern, IntoPyObjectExt}; use super::to_numpy_series::series_to_numpy; use super::utils::{ @@ -78,7 +78,7 @@ fn try_df_to_numpy_view(py: Python, df: &DataFrame, allow_nulls: bool) -> Option return None; } - let owner = PyDataFrame::from(df.clone()).into_py(py); // Keep the DataFrame memory alive. + let owner = PyDataFrame::from(df.clone()).into_py_any(py).ok()?; // Keep the DataFrame memory alive. let arr = match first_dtype { dt if dt.is_numeric() => { @@ -187,7 +187,7 @@ where let first_slice = ca.data_views().next().unwrap(); let start_ptr = first_slice.as_ptr(); - let np_dtype = T::Native::get_dtype_bound(py); + let np_dtype = T::Native::get_dtype(py); let dims = [first_slice.len(), df.width()].into_dimension(); unsafe { @@ -245,7 +245,7 @@ fn try_df_to_numpy_numeric_supertype( let np_array = match st { dt if dt.is_numeric() => with_match_physical_numpy_polars_type!(dt, |$T| { - df.to_ndarray::<$T>(order).ok()?.into_pyarray_bound(py).into_py(py) + df.to_ndarray::<$T>(order).ok()?.into_pyarray(py).into_py_any(py).ok()? }), _ => return None, }; @@ -273,21 +273,19 @@ fn df_columns_to_numpy( arr.call_method1(py, intern!(py, "__getitem__"), (idx,)) .unwrap() }); - arr = PyArray1::from_iter_bound(py, subarrays).into_py(py); + arr = PyArray1::from_iter(py, subarrays).into_py_any(py).unwrap(); } arr }); - let numpy = PyModule::import_bound(py, intern!(py, "numpy"))?; + let numpy = PyModule::import(py, intern!(py, "numpy"))?; let np_array = match order { IndexOrder::C => numpy - .getattr(intern!(py, "column_stack")) - .unwrap() - .call1((PyList::new_bound(py, np_arrays),))?, + .getattr(intern!(py, "column_stack"))? + .call1((PyList::new(py, np_arrays)?,))?, IndexOrder::Fortran => numpy - .getattr(intern!(py, "vstack")) - .unwrap() - .call1((PyList::new_bound(py, np_arrays),))? + .getattr(intern!(py, "vstack"))? + .call1((PyList::new(py, np_arrays)?,))? .getattr(intern!(py, "T"))?, }; diff --git a/crates/polars-python/src/interop/numpy/to_numpy_series.rs b/crates/polars-python/src/interop/numpy/to_numpy_series.rs index 81a3d68d088d..8974a2d4364d 100644 --- a/crates/polars-python/src/interop/numpy/to_numpy_series.rs +++ b/crates/polars-python/src/interop/numpy/to_numpy_series.rs @@ -4,8 +4,8 @@ use numpy::npyffi::flags; use numpy::{Element, PyArray1}; use polars_core::prelude::*; use pyo3::exceptions::PyRuntimeError; -use pyo3::intern; use pyo3::prelude::*; +use pyo3::{intern, IntoPyObjectExt}; use super::to_numpy_df::df_to_numpy; use super::utils::{ @@ -119,7 +119,7 @@ fn series_to_numpy_view_recursive(py: Python, s: Series, writable: bool) -> PyOb fn numeric_series_to_numpy_view(py: Python, s: Series, writable: bool) -> PyObject { let dims = [s.len()].into_dimension(); with_match_physical_numpy_polars_type!(s.dtype(), |$T| { - let np_dtype = <$T as PolarsNumericType>::Native::get_dtype_bound(py); + let np_dtype = <$T as PolarsNumericType>::Native::get_dtype(py); let ca: &ChunkedArray<$T> = s.unpack::<$T>().unwrap(); let flags = if writable { flags::NPY_ARRAY_FARRAY @@ -136,7 +136,7 @@ fn numeric_series_to_numpy_view(py: Python, s: Series, writable: bool) -> PyObje dims, flags, slice.as_ptr() as _, - PySeries::from(s).into_py(py), // Keep the Series memory alive., + PySeries::from(s).into_py_any(py).unwrap(), // Keep the Series memory alive., ) } }) @@ -162,7 +162,7 @@ fn temporal_series_to_numpy_view(py: Python, s: Series, writable: bool) -> PyObj dims, flags, slice.as_ptr() as _, - PySeries::from(s).into_py(py), // Keep the Series memory alive., + PySeries::from(s).into_py_any(py).unwrap(), // Keep the Series memory alive., ) } } @@ -176,7 +176,7 @@ fn array_series_to_numpy_view(py: Python, s: &Series, writable: bool) -> PyObjec let DataType::Array(_, width) = s.dtype() else { unreachable!() }; - reshape_numpy_array(py, np_array_flat, ca.len(), *width) + reshape_numpy_array(py, np_array_flat, ca.len(), *width).unwrap() } /// Convert a Series to a NumPy ndarray, copying data in the process. @@ -231,28 +231,30 @@ fn series_to_numpy_with_copy(py: Python, s: &Series, writable: bool) -> PyObject }, Time => { let ca = s.time().unwrap(); - let values = time_to_pyobject_iter(ca).map(|v| v.into_py(py)); - PyArray1::from_iter_bound(py, values).into_py(py) + let values = time_to_pyobject_iter(ca).map(|v| v.into_py_any(py).unwrap()); + PyArray1::from_iter(py, values).into_py_any(py).unwrap() }, String => { let ca = s.str().unwrap(); - let values = ca.iter().map(|s| s.into_py(py)); - PyArray1::from_iter_bound(py, values).into_py(py) + let values = ca.iter().map(|s| s.into_py_any(py).unwrap()); + PyArray1::from_iter(py, values).into_py_any(py).unwrap() }, Binary => { let ca = s.binary().unwrap(); - let values = ca.iter().map(|s| s.into_py(py)); - PyArray1::from_iter_bound(py, values).into_py(py) + let values = ca.iter().map(|s| s.into_py_any(py).unwrap()); + PyArray1::from_iter(py, values).into_py_any(py).unwrap() }, Categorical(_, _) | Enum(_, _) => { let ca = s.categorical().unwrap(); - let values = ca.iter_str().map(|s| s.into_py(py)); - PyArray1::from_iter_bound(py, values).into_py(py) + let values = ca.iter_str().map(|s| s.into_py_any(py).unwrap()); + PyArray1::from_iter(py, values).into_py_any(py).unwrap() }, Decimal(_, _) => { let ca = s.decimal().unwrap(); - let values = decimal_to_pyobject_iter(py, ca).map(|v| v.into_py(py)); - PyArray1::from_iter_bound(py, values).into_py(py) + let values = decimal_to_pyobject_iter(py, ca) + .unwrap() + .map(|v| v.into_py_any(py).unwrap()); + PyArray1::from_iter(py, values).into_py_any(py).unwrap() }, List(_) => list_series_to_numpy(py, s, writable), Array(_, _) => array_series_to_numpy(py, s, writable), @@ -267,13 +269,13 @@ fn series_to_numpy_with_copy(py: Python, s: &Series, writable: bool) -> PyObject .as_any() .downcast_ref::>() .unwrap(); - let values = ca.iter().map(|v| v.to_object(py)); - PyArray1::from_iter_bound(py, values).into_py(py) + let values = ca.iter().map(|v| v.into_py_any(py).unwrap()); + PyArray1::from_iter(py, values).into_py_any(py).unwrap() }, Null => { let n = s.len(); let values = std::iter::repeat(f32::NAN).take(n); - PyArray1::from_iter_bound(py, values).into_py(py) + PyArray1::from_iter(py, values).into_py_any(py).unwrap() }, Unknown(_) | BinaryOffset => unreachable!(), } @@ -289,14 +291,16 @@ where let ca: &ChunkedArray = s.as_ref().as_ref(); if s.null_count() == 0 { let values = ca.into_no_null_iter(); - PyArray1::::from_iter_bound(py, values).into_py(py) + PyArray1::::from_iter(py, values) + .into_py_any(py) + .unwrap() } else { let mapper = |opt_v: Option| match opt_v { Some(v) => NumCast::from(v).unwrap(), None => U::nan(), }; let values = ca.iter().map(mapper); - PyArray1::from_iter_bound(py, values).into_py(py) + PyArray1::from_iter(py, values).into_py_any(py).unwrap() } } /// Convert booleans to u8 if no nulls are present, otherwise convert to objects. @@ -304,10 +308,12 @@ fn boolean_series_to_numpy(py: Python, s: &Series) -> PyObject { let ca = s.bool().unwrap(); if s.null_count() == 0 { let values = ca.into_no_null_iter(); - PyArray1::::from_iter_bound(py, values).into_py(py) + PyArray1::::from_iter(py, values) + .into_py_any(py) + .unwrap() } else { - let values = ca.iter().map(|opt_v| opt_v.into_py(py)); - PyArray1::from_iter_bound(py, values).into_py(py) + let values = ca.iter().map(|opt_v| opt_v.into_py_any(py).unwrap()); + PyArray1::from_iter(py, values).into_py_any(py).unwrap() } } /// Convert dates directly to i64 with i64::MIN representing a null value. @@ -320,7 +326,9 @@ fn date_series_to_numpy(py: Python, s: &Series) -> PyObject { if s.null_count() == 0 { let mapper = |v: i32| (v as i64).into(); let values = ca.into_no_null_iter().map(mapper); - PyArray1::>::from_iter_bound(py, values).into_py(py) + PyArray1::>::from_iter(py, values) + .into_py_any(py) + .unwrap() } else { let mapper = |opt_v: Option| { match opt_v { @@ -330,7 +338,9 @@ fn date_series_to_numpy(py: Python, s: &Series) -> PyObject { .into() }; let values = ca.iter().map(mapper); - PyArray1::>::from_iter_bound(py, values).into_py(py) + PyArray1::>::from_iter(py, values) + .into_py_any(py) + .unwrap() } } /// Convert datetimes and durations with i64::MIN representing a null value. @@ -341,7 +351,9 @@ where let s_phys = s.to_physical_repr(); let ca = s_phys.i64().unwrap(); let values = ca.iter().map(|v| v.unwrap_or(i64::MIN).into()); - PyArray1::::from_iter_bound(py, values).into_py(py) + PyArray1::::from_iter(py, values) + .into_py_any(py) + .unwrap() } fn list_series_to_numpy(py: Python, s: &Series, writable: bool) -> PyObject { let ca = s.list().unwrap(); @@ -350,7 +362,7 @@ fn list_series_to_numpy(py: Python, s: &Series, writable: bool) -> PyObject { None => py.None(), Some(s) => series_to_numpy(py, s.as_ref(), writable, true).unwrap(), }); - PyArray1::from_iter_bound(py, iter).into_py(py) + PyArray1::from_iter(py, iter).into_py_any(py).unwrap() } /// Convert arrays by flattening first, converting the flat Series, and then reshaping. fn array_series_to_numpy(py: Python, s: &Series, writable: bool) -> PyObject { @@ -362,5 +374,5 @@ fn array_series_to_numpy(py: Python, s: &Series, writable: bool) -> PyObject { let DataType::Array(_, width) = s.dtype() else { unreachable!() }; - reshape_numpy_array(py, np_array_flat, ca.len(), *width) + reshape_numpy_array(py, np_array_flat, ca.len(), *width).unwrap() } diff --git a/crates/polars-python/src/interop/numpy/utils.rs b/crates/polars-python/src/interop/numpy/utils.rs index dc0a47fcac26..dfaa805e0352 100644 --- a/crates/polars-python/src/interop/numpy/utils.rs +++ b/crates/polars-python/src/interop/numpy/utils.rs @@ -74,26 +74,22 @@ pub(super) fn reshape_numpy_array( arr: PyObject, height: usize, width: usize, -) -> PyObject { +) -> PyResult { let shape = arr - .getattr(py, intern!(py, "shape")) - .unwrap() - .extract::>(py) - .unwrap(); + .getattr(py, intern!(py, "shape"))? + .extract::>(py)?; if shape.len() == 1 { // In this case, we can avoid allocating a Vec. let new_shape = (height, width); arr.call_method1(py, intern!(py, "reshape"), new_shape) - .unwrap() } else { let mut new_shape_vec = vec![height, width]; for v in &shape[1..] { new_shape_vec.push(*v) } - let new_shape = PyTuple::new_bound(py, new_shape_vec); + let new_shape = PyTuple::new(py, new_shape_vec)?; arr.call_method1(py, intern!(py, "reshape"), new_shape) - .unwrap() } } @@ -105,23 +101,21 @@ pub(super) fn polars_dtype_to_np_temporal_dtype<'a>( use numpy::datetime::{units, Datetime, Timedelta}; match dtype { DataType::Datetime(TimeUnit::Milliseconds, _) => { - Datetime::::get_dtype_bound(py) + Datetime::::get_dtype(py) }, DataType::Datetime(TimeUnit::Microseconds, _) => { - Datetime::::get_dtype_bound(py) + Datetime::::get_dtype(py) }, DataType::Datetime(TimeUnit::Nanoseconds, _) => { - Datetime::::get_dtype_bound(py) + Datetime::::get_dtype(py) }, DataType::Duration(TimeUnit::Milliseconds) => { - Timedelta::::get_dtype_bound(py) + Timedelta::::get_dtype(py) }, DataType::Duration(TimeUnit::Microseconds) => { - Timedelta::::get_dtype_bound(py) - }, - DataType::Duration(TimeUnit::Nanoseconds) => { - Timedelta::::get_dtype_bound(py) + Timedelta::::get_dtype(py) }, + DataType::Duration(TimeUnit::Nanoseconds) => Timedelta::::get_dtype(py), _ => panic!("only Datetime/Duration inputs supported, got {}", dtype), } } diff --git a/crates/polars-python/src/lazyframe/general.rs b/crates/polars-python/src/lazyframe/general.rs index 3c835d063f77..9b1ab31c74a1 100644 --- a/crates/polars-python/src/lazyframe/general.rs +++ b/crates/polars-python/src/lazyframe/general.rs @@ -238,7 +238,7 @@ impl PyLazyFrame { let f = |schema: Schema| { let iter = schema.iter_names().map(|s| s.as_str()); Python::with_gil(|py| { - let names = PyList::new_bound(py, iter); + let names = PyList::new(py, iter).unwrap(); let out = lambda.call1(py, (names,)).expect("python function failed"); let new_names = out @@ -662,7 +662,7 @@ impl PyLazyFrame { }, Err(err) => { lambda - .call1(py, (PyErr::from(err).to_object(py),)) + .call1(py, (PyErr::from(err),)) .map_err(|err| err.restore(py)) .ok(); }, @@ -1296,18 +1296,18 @@ impl PyLazyFrame { self.ldf.clone().into() } - fn collect_schema(&mut self, py: Python) -> PyResult { + fn collect_schema<'py>(&mut self, py: Python<'py>) -> PyResult> { let schema = py .allow_threads(|| self.ldf.collect_schema()) .map_err(PyPolarsErr::from)?; - let schema_dict = PyDict::new_bound(py); + let schema_dict = PyDict::new(py); schema.iter_fields().for_each(|fld| { schema_dict - .set_item(fld.name().as_str(), Wrap(fld.dtype().clone())) + .set_item(fld.name().as_str(), &Wrap(fld.dtype().clone())) .unwrap() }); - Ok(schema_dict.to_object(py)) + Ok(schema_dict) } fn unnest(&self, columns: Vec) -> Self { diff --git a/crates/polars-python/src/lazyframe/serde.rs b/crates/polars-python/src/lazyframe/serde.rs index fa51fa9efb37..82e2d4c87f5d 100644 --- a/crates/polars-python/src/lazyframe/serde.rs +++ b/crates/polars-python/src/lazyframe/serde.rs @@ -13,13 +13,13 @@ use crate::prelude::*; #[pymethods] #[allow(clippy::should_implement_trait)] impl PyLazyFrame { - fn __getstate__(&self, py: Python) -> PyResult { + fn __getstate__<'py>(&self, py: Python<'py>) -> PyResult> { // Used in pickle/pickling let mut writer: Vec = vec![]; ciborium::ser::into_writer(&self.ldf.logical_plan, &mut writer) .map_err(|e| PyPolarsErr::Other(format!("{}", e)))?; - Ok(PyBytes::new_bound(py, &writer).to_object(py)) + Ok(PyBytes::new(py, &writer)) } fn __setstate__(&mut self, py: Python, state: PyObject) -> PyResult<()> { diff --git a/crates/polars-python/src/lazyframe/visit.rs b/crates/polars-python/src/lazyframe/visit.rs index 27633e401301..4198b54c7ad7 100644 --- a/crates/polars-python/src/lazyframe/visit.rs +++ b/crates/polars-python/src/lazyframe/visit.rs @@ -6,7 +6,7 @@ use polars_plan::prelude::expr_ir::ExprIR; use polars_plan::prelude::{AExpr, PythonOptions, PythonScanSource}; use polars_utils::arena::{Arena, Node}; use pyo3::prelude::*; -use pyo3::types::PyList; +use pyo3::types::{PyDict, PyList}; use super::visitor::{expr_nodes, nodes}; use super::PyLazyFrame; @@ -89,37 +89,32 @@ impl NodeTraverser { this_node.copy_exprs(&mut self.expr_scratch); } - fn scratch_to_list(&mut self) -> PyObject { - Python::with_gil(|py| { - PyList::new_bound(py, self.scratch.drain(..).map(|node| node.0)).to_object(py) - }) + fn scratch_to_list<'py>(&mut self, py: Python<'py>) -> PyResult> { + PyList::new(py, self.scratch.drain(..).map(|node| node.0)) } - fn expr_to_list(&mut self) -> PyObject { - Python::with_gil(|py| { - PyList::new_bound( - py, - self.expr_scratch - .drain(..) - .map(|e| PyExprIR::from(e).into_py(py)), - ) - .to_object(py) - }) + fn expr_to_list<'py>(&mut self, py: Python<'py>) -> PyResult> { + PyList::new( + py, + self.expr_scratch + .drain(..) + .map(|e| PyExprIR::from(e).into_pyobject(py).unwrap()), + ) } } #[pymethods] impl NodeTraverser { /// Get expression nodes - fn get_exprs(&mut self) -> PyObject { + fn get_exprs<'py>(&mut self, py: Python<'py>) -> PyResult> { self.fill_expressions(); - self.expr_to_list() + self.expr_to_list(py) } /// Get input nodes - fn get_inputs(&mut self) -> PyObject { + fn get_inputs<'py>(&mut self, py: Python<'py>) -> PyResult> { self.fill_inputs(); - self.scratch_to_list() + self.scratch_to_list(py) } /// The current version of the IR @@ -128,14 +123,14 @@ impl NodeTraverser { } /// Get Schema of current node as python dict - fn get_schema(&self, py: Python<'_>) -> PyObject { + fn get_schema<'py>(&self, py: Python<'py>) -> PyResult> { let lp_arena = self.lp_arena.lock().unwrap(); let schema = lp_arena.get(self.root).schema(&lp_arena); - Wrap(&**schema).into_py(py) + Wrap(&**schema).into_pyobject(py) } /// Get expression dtype of expr_node, the schema used is that of the current root node - fn get_dtype(&self, expr_node: usize, py: Python<'_>) -> PyResult { + fn get_dtype<'py>(&self, expr_node: usize, py: Python<'py>) -> PyResult> { let expr_node = Node(expr_node); let lp_arena = self.lp_arena.lock().unwrap(); let schema = lp_arena.get(self.root).schema(&lp_arena); @@ -144,7 +139,7 @@ impl NodeTraverser { .get(expr_node) .to_field(&schema, Context::Default, &expr_arena) .map_err(PyPolarsErr::from)?; - Ok(Wrap(field.dtype).to_object(py)) + Wrap(field.dtype).into_pyobject(py) } /// Set the current node in the plan. diff --git a/crates/polars-python/src/lazyframe/visitor/expr_nodes.rs b/crates/polars-python/src/lazyframe/visitor/expr_nodes.rs index ab5c82843379..2958302811b8 100644 --- a/crates/polars-python/src/lazyframe/visitor/expr_nodes.rs +++ b/crates/polars-python/src/lazyframe/visitor/expr_nodes.rs @@ -1,4 +1,3 @@ -use polars::datatypes::TimeUnit; #[cfg(feature = "iejoin")] use polars::prelude::InequalityOperator; use polars::series::ops::NullBehavior; @@ -17,6 +16,8 @@ use polars_time::prelude::RollingGroupOptions; use polars_time::{Duration, DynamicGroupOptions}; use pyo3::exceptions::PyNotImplementedError; use pyo3::prelude::*; +use pyo3::types::PyTuple; +use pyo3::IntoPyObjectExt; use crate::series::PySeries; use crate::Wrap; @@ -75,8 +76,12 @@ impl PyOperator { } } -impl IntoPy for Wrap { - fn into_py(self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for Wrap { + type Target = PyOperator; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { match self.0 { Operator::Eq => PyOperator::Eq, Operator::EqValidity => PyOperator::EqValidity, @@ -99,20 +104,24 @@ impl IntoPy for Wrap { Operator::LogicalAnd => PyOperator::LogicalAnd, Operator::LogicalOr => PyOperator::LogicalOr, } - .into_py(py) + .into_pyobject(py) } } #[cfg(feature = "iejoin")] -impl IntoPy for Wrap { - fn into_py(self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for Wrap { + type Target = PyOperator; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { match self.0 { InequalityOperator::Lt => PyOperator::Lt, InequalityOperator::LtEq => PyOperator::LtEq, InequalityOperator::Gt => PyOperator::Gt, InequalityOperator::GtEq => PyOperator::GtEq, } - .into_py(py) + .into_pyobject(py) } } @@ -255,12 +264,6 @@ impl PyTemporalFunction { } } -impl IntoPy for Wrap { - fn into_py(self, py: Python<'_>) -> PyObject { - self.0.to_ascii().into_py(py) - } -} - #[pyclass] pub struct BinaryExpr { #[pyo3(get)] @@ -390,14 +393,17 @@ pub struct PyWindowMapping { #[pymethods] impl PyWindowMapping { #[getter] - fn kind(&self, py: Python<'_>) -> PyResult { - let result: &str = self.inner.into(); - Ok(result.into_py(py)) + fn kind(&self) -> &str { + self.inner.into() } } -impl IntoPy for Wrap { - fn into_py(self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for Wrap { + type Target = PyTuple; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { ( self.0.months(), self.0.weeks(), @@ -406,7 +412,7 @@ impl IntoPy for Wrap { self.0.parsed_int, self.0.negative(), ) - .into_py(py) + .into_pyobject(py) } } @@ -418,24 +424,23 @@ pub struct PyRollingGroupOptions { #[pymethods] impl PyRollingGroupOptions { #[getter] - fn index_column(&self, py: Python<'_>) -> PyResult { - Ok(self.inner.index_column.to_object(py)) + fn index_column(&self) -> &str { + self.inner.index_column.as_str() } #[getter] - fn period(&self, py: Python<'_>) -> PyResult { - Ok(Wrap(self.inner.period).into_py(py)) + fn period(&self) -> Wrap { + Wrap(self.inner.period) } #[getter] - fn offset(&self, py: Python<'_>) -> PyResult { - Ok(Wrap(self.inner.offset).into_py(py)) + fn offset(&self) -> Wrap { + Wrap(self.inner.offset) } #[getter] - fn closed_window(&self, py: Python<'_>) -> PyResult { - let result: &str = self.inner.closed_window.into(); - Ok(result.to_object(py)) + fn closed_window(&self) -> &str { + self.inner.closed_window.into() } } @@ -447,45 +452,42 @@ pub struct PyDynamicGroupOptions { #[pymethods] impl PyDynamicGroupOptions { #[getter] - fn index_column(&self, py: Python<'_>) -> PyResult { - Ok(self.inner.index_column.to_object(py)) + fn index_column(&self) -> &str { + self.inner.index_column.as_str() } #[getter] - fn every(&self, py: Python<'_>) -> PyResult { - Ok(Wrap(self.inner.every).into_py(py)) + fn every(&self) -> Wrap { + Wrap(self.inner.every) } #[getter] - fn period(&self, py: Python<'_>) -> PyResult { - Ok(Wrap(self.inner.period).into_py(py)) + fn period(&self) -> Wrap { + Wrap(self.inner.period) } #[getter] - fn offset(&self, py: Python<'_>) -> PyResult { - Ok(Wrap(self.inner.offset).into_py(py)) + fn offset(&self) -> Wrap { + Wrap(self.inner.offset) } #[getter] - fn label(&self, py: Python<'_>) -> PyResult { - let result: &str = self.inner.label.into(); - Ok(result.to_object(py)) + fn label(&self) -> &str { + self.inner.label.into() } #[getter] - fn include_boundaries(&self, py: Python<'_>) -> PyResult { - Ok(self.inner.include_boundaries.into_py(py)) + fn include_boundaries(&self) -> bool { + self.inner.include_boundaries } #[getter] - fn closed_window(&self, py: Python<'_>) -> PyResult { - let result: &str = self.inner.closed_window.into(); - Ok(result.to_object(py)) + fn closed_window(&self) -> &str { + self.inner.closed_window.into() } #[getter] - fn start_by(&self, py: Python<'_>) -> PyResult { - let result: &str = self.inner.start_by.into(); - Ok(result.to_object(py)) + fn start_by(&self) -> &str { + self.inner.start_by.into() } } @@ -503,108 +505,105 @@ impl PyGroupbyOptions { #[pymethods] impl PyGroupbyOptions { #[getter] - fn slice(&self, py: Python<'_>) -> PyResult { - Ok(self - .inner - .slice - .map_or_else(|| py.None(), |f| f.to_object(py))) + fn slice(&self) -> Option<(i64, usize)> { + self.inner.slice } #[getter] - fn dynamic(&self, py: Python<'_>) -> PyResult { - Ok(self.inner.dynamic.as_ref().map_or_else( - || py.None(), - |f| PyDynamicGroupOptions { inner: f.clone() }.into_py(py), - )) + fn dynamic(&self) -> Option { + self.inner + .dynamic + .as_ref() + .map(|f| PyDynamicGroupOptions { inner: f.clone() }) } #[getter] - fn rolling(&self, py: Python<'_>) -> PyResult { - Ok(self.inner.rolling.as_ref().map_or_else( - || py.None(), - |f| PyRollingGroupOptions { inner: f.clone() }.into_py(py), - )) + fn rolling(&self) -> Option { + self.inner + .rolling + .as_ref() + .map(|f| PyRollingGroupOptions { inner: f.clone() }) } } pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { - let result = match expr { - AExpr::Explode(_) => return Err(PyNotImplementedError::new_err("explode")), + match expr { + AExpr::Explode(_) => Err(PyNotImplementedError::new_err("explode")), AExpr::Alias(inner, name) => Alias { expr: inner.0, - name: name.to_object(py), + name: name.into_py_any(py)?, } - .into_py(py), + .into_py_any(py), AExpr::Column(name) => Column { - name: name.to_object(py), + name: name.into_py_any(py)?, } - .into_py(py), + .into_py_any(py), AExpr::Literal(lit) => { use LiteralValue::*; - let dtype: PyObject = Wrap(lit.get_datatype()).to_object(py); + let dtype: PyObject = Wrap(lit.get_datatype()).into_py_any(py)?; match lit { Float(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Float32(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Float64(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Int(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Int8(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Int16(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Int32(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Int64(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Int128(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, UInt8(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, UInt16(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, UInt32(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, UInt64(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Boolean(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, StrCat(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, String(v) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Null => Literal { @@ -615,40 +614,40 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { Range { .. } => return Err(PyNotImplementedError::new_err("range literal")), OtherScalar { .. } => return Err(PyNotImplementedError::new_err("scalar literal")), Date(..) | DateTime(..) | Decimal(..) => Literal { - value: Wrap(lit.to_any_value().unwrap()).to_object(py), + value: Wrap(lit.to_any_value().unwrap()).into_py_any(py)?, dtype, }, Duration(v, _) => Literal { - value: v.to_object(py), + value: v.into_py_any(py)?, dtype, }, Time(ns) => Literal { - value: ns.to_object(py), + value: ns.into_py_any(py)?, dtype, }, Series(s) => Literal { - value: PySeries::new((**s).clone()).into_py(py), + value: PySeries::new((**s).clone()).into_py_any(py)?, dtype, }, } } - .into_py(py), + .into_py_any(py), AExpr::BinaryExpr { left, op, right } => BinaryExpr { left: left.0, - op: Wrap(*op).into_py(py), + op: Wrap(*op).into_py_any(py)?, right: right.0, } - .into_py(py), + .into_py_any(py), AExpr::Cast { expr, dtype, options, } => Cast { expr: expr.0, - dtype: Wrap(dtype.clone()).to_object(py), + dtype: Wrap(dtype.clone()).into_py_any(py)?, options: *options as u8, } - .into_py(py), + .into_py_any(py), AExpr::Sort { expr, options } => Sort { expr: expr.0, options: ( @@ -657,7 +656,7 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { options.descending, ), } - .into_py(py), + .into_py_any(py), AExpr::Gather { expr, idx, @@ -667,12 +666,12 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { idx: idx.0, scalar: *returns_scalar, } - .into_py(py), + .into_py_any(py), AExpr::Filter { input, by } => Filter { input: input.0, by: by.0, } - .into_py(py), + .into_py_any(py), AExpr::SortBy { expr, by, @@ -686,51 +685,51 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { sort_options.descending.clone(), ), } - .into_py(py), + .into_py_any(py), AExpr::Agg(aggexpr) => match aggexpr { IRAggExpr::Min { input, propagate_nans, } => Agg { - name: "min".to_object(py), + name: "min".into_py_any(py)?, arguments: vec![input.0], - options: propagate_nans.to_object(py), + options: propagate_nans.into_py_any(py)?, }, IRAggExpr::Max { input, propagate_nans, } => Agg { - name: "max".to_object(py), + name: "max".into_py_any(py)?, arguments: vec![input.0], - options: propagate_nans.to_object(py), + options: propagate_nans.into_py_any(py)?, }, IRAggExpr::Median(n) => Agg { - name: "median".to_object(py), + name: "median".into_py_any(py)?, arguments: vec![n.0], options: py.None(), }, IRAggExpr::NUnique(n) => Agg { - name: "n_unique".to_object(py), + name: "n_unique".into_py_any(py)?, arguments: vec![n.0], options: py.None(), }, IRAggExpr::First(n) => Agg { - name: "first".to_object(py), + name: "first".into_py_any(py)?, arguments: vec![n.0], options: py.None(), }, IRAggExpr::Last(n) => Agg { - name: "last".to_object(py), + name: "last".into_py_any(py)?, arguments: vec![n.0], options: py.None(), }, IRAggExpr::Mean(n) => Agg { - name: "mean".to_object(py), + name: "mean".into_py_any(py)?, arguments: vec![n.0], options: py.None(), }, IRAggExpr::Implode(n) => Agg { - name: "implode".to_object(py), + name: "implode".into_py_any(py)?, arguments: vec![n.0], options: py.None(), }, @@ -739,37 +738,37 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { quantile, method: interpol, } => Agg { - name: "quantile".to_object(py), + name: "quantile".into_py_any(py)?, arguments: vec![expr.0, quantile.0], - options: Into::<&str>::into(interpol).to_object(py), + options: Into::<&str>::into(interpol).into_py_any(py)?, }, IRAggExpr::Sum(n) => Agg { - name: "sum".to_object(py), + name: "sum".into_py_any(py)?, arguments: vec![n.0], options: py.None(), }, IRAggExpr::Count(n, include_null) => Agg { - name: "count".to_object(py), + name: "count".into_py_any(py)?, arguments: vec![n.0], - options: include_null.to_object(py), + options: include_null.into_py_any(py)?, }, IRAggExpr::Std(n, ddof) => Agg { - name: "std".to_object(py), + name: "std".into_py_any(py)?, arguments: vec![n.0], - options: ddof.to_object(py), + options: ddof.into_py_any(py)?, }, IRAggExpr::Var(n, ddof) => Agg { - name: "var".to_object(py), + name: "var".into_py_any(py)?, arguments: vec![n.0], - options: ddof.to_object(py), + options: ddof.into_py_any(py)?, }, IRAggExpr::AggGroups(n) => Agg { - name: "agg_groups".to_object(py), + name: "agg_groups".into_py_any(py)?, arguments: vec![n.0], options: py.None(), }, } - .into_py(py), + .into_py_any(py), AExpr::Ternary { predicate, truthy, @@ -779,10 +778,8 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { truthy: truthy.0, falsy: falsy.0, } - .into_py(py), - AExpr::AnonymousFunction { .. } => { - return Err(PyNotImplementedError::new_err("anonymousfunction")) - }, + .into_py_any(py), + AExpr::AnonymousFunction { .. } => Err(PyNotImplementedError::new_err("anonymousfunction")), AExpr::Function { input, function, @@ -811,163 +808,124 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { delimiter, ignore_nulls, } => ( - PyStringFunction::ConcatHorizontal.into_py(py), + PyStringFunction::ConcatHorizontal, delimiter.as_str(), ignore_nulls, ) - .to_object(py), + .into_py_any(py), StringFunction::ConcatVertical { delimiter, ignore_nulls, } => ( - PyStringFunction::ConcatVertical.into_py(py), + PyStringFunction::ConcatVertical, delimiter.as_str(), ignore_nulls, ) - .to_object(py), + .into_py_any(py), #[cfg(feature = "regex")] StringFunction::Contains { literal, strict } => { - (PyStringFunction::Contains.into_py(py), literal, strict).to_object(py) + (PyStringFunction::Contains, literal, strict).into_py_any(py) }, StringFunction::CountMatches(literal) => { - (PyStringFunction::CountMatches.into_py(py), literal).to_object(py) - }, - StringFunction::EndsWith => { - (PyStringFunction::EndsWith.into_py(py),).to_object(py) + (PyStringFunction::CountMatches, literal).into_py_any(py) }, + StringFunction::EndsWith => (PyStringFunction::EndsWith,).into_py_any(py), StringFunction::Extract(group_index) => { - (PyStringFunction::Extract.into_py(py), group_index).to_object(py) - }, - StringFunction::ExtractAll => { - (PyStringFunction::ExtractAll.into_py(py),).to_object(py) + (PyStringFunction::Extract, group_index).into_py_any(py) }, + StringFunction::ExtractAll => (PyStringFunction::ExtractAll,).into_py_any(py), #[cfg(feature = "extract_groups")] StringFunction::ExtractGroups { dtype, pat } => ( - PyStringFunction::ExtractGroups.into_py(py), - Wrap(dtype.clone()).to_object(py), + PyStringFunction::ExtractGroups, + &Wrap(dtype.clone()), pat.as_str(), ) - .to_object(py), + .into_py_any(py), #[cfg(feature = "regex")] StringFunction::Find { literal, strict } => { - (PyStringFunction::Find.into_py(py), literal, strict).to_object(py) + (PyStringFunction::Find, literal, strict).into_py_any(py) }, StringFunction::ToInteger(strict) => { - (PyStringFunction::ToInteger.into_py(py), strict).to_object(py) - }, - StringFunction::LenBytes => { - (PyStringFunction::LenBytes.into_py(py),).to_object(py) - }, - StringFunction::LenChars => { - (PyStringFunction::LenChars.into_py(py),).to_object(py) - }, - StringFunction::Lowercase => { - (PyStringFunction::Lowercase.into_py(py),).to_object(py) + (PyStringFunction::ToInteger, strict).into_py_any(py) }, + StringFunction::LenBytes => (PyStringFunction::LenBytes,).into_py_any(py), + StringFunction::LenChars => (PyStringFunction::LenChars,).into_py_any(py), + StringFunction::Lowercase => (PyStringFunction::Lowercase,).into_py_any(py), #[cfg(feature = "extract_jsonpath")] StringFunction::JsonDecode { dtype: _, infer_schema_len, - } => (PyStringFunction::JsonDecode.into_py(py), infer_schema_len).to_object(py), + } => (PyStringFunction::JsonDecode, infer_schema_len).into_py_any(py), #[cfg(feature = "extract_jsonpath")] StringFunction::JsonPathMatch => { - (PyStringFunction::JsonPathMatch.into_py(py),).to_object(py) + (PyStringFunction::JsonPathMatch,).into_py_any(py) }, #[cfg(feature = "regex")] StringFunction::Replace { n, literal } => { - (PyStringFunction::Replace.into_py(py), n, literal).to_object(py) - }, - StringFunction::Reverse => { - (PyStringFunction::Reverse.into_py(py),).to_object(py) + (PyStringFunction::Replace, n, literal).into_py_any(py) }, + StringFunction::Reverse => (PyStringFunction::Reverse,).into_py_any(py), StringFunction::PadStart { length, fill_char } => { - (PyStringFunction::PadStart.into_py(py), length, fill_char).to_object(py) + (PyStringFunction::PadStart, length, fill_char).into_py_any(py) }, StringFunction::PadEnd { length, fill_char } => { - (PyStringFunction::PadEnd.into_py(py), length, fill_char).to_object(py) - }, - StringFunction::Slice => (PyStringFunction::Slice.into_py(py),).to_object(py), - StringFunction::Head => (PyStringFunction::Head.into_py(py),).to_object(py), - StringFunction::Tail => (PyStringFunction::Tail.into_py(py),).to_object(py), - StringFunction::HexEncode => { - (PyStringFunction::HexEncode.into_py(py),).to_object(py) + (PyStringFunction::PadEnd, length, fill_char).into_py_any(py) }, + StringFunction::Slice => (PyStringFunction::Slice,).into_py_any(py), + StringFunction::Head => (PyStringFunction::Head,).into_py_any(py), + StringFunction::Tail => (PyStringFunction::Tail,).into_py_any(py), + StringFunction::HexEncode => (PyStringFunction::HexEncode,).into_py_any(py), #[cfg(feature = "binary_encoding")] StringFunction::HexDecode(strict) => { - (PyStringFunction::HexDecode.into_py(py), strict).to_object(py) + (PyStringFunction::HexDecode, strict).into_py_any(py) }, StringFunction::Base64Encode => { - (PyStringFunction::Base64Encode.into_py(py),).to_object(py) + (PyStringFunction::Base64Encode,).into_py_any(py) }, #[cfg(feature = "binary_encoding")] StringFunction::Base64Decode(strict) => { - (PyStringFunction::Base64Decode.into_py(py), strict).to_object(py) - }, - StringFunction::StartsWith => { - (PyStringFunction::StartsWith.into_py(py),).to_object(py) - }, - StringFunction::StripChars => { - (PyStringFunction::StripChars.into_py(py),).to_object(py) + (PyStringFunction::Base64Decode, strict).into_py_any(py) }, + StringFunction::StartsWith => (PyStringFunction::StartsWith,).into_py_any(py), + StringFunction::StripChars => (PyStringFunction::StripChars,).into_py_any(py), StringFunction::StripCharsStart => { - (PyStringFunction::StripCharsStart.into_py(py),).to_object(py) + (PyStringFunction::StripCharsStart,).into_py_any(py) }, StringFunction::StripCharsEnd => { - (PyStringFunction::StripCharsEnd.into_py(py),).to_object(py) - }, - StringFunction::StripPrefix => { - (PyStringFunction::StripPrefix.into_py(py),).to_object(py) - }, - StringFunction::StripSuffix => { - (PyStringFunction::StripSuffix.into_py(py),).to_object(py) + (PyStringFunction::StripCharsEnd,).into_py_any(py) }, + StringFunction::StripPrefix => (PyStringFunction::StripPrefix,).into_py_any(py), + StringFunction::StripSuffix => (PyStringFunction::StripSuffix,).into_py_any(py), StringFunction::SplitExact { n, inclusive } => { - (PyStringFunction::SplitExact.into_py(py), n, inclusive).to_object(py) - }, - StringFunction::SplitN(n) => { - (PyStringFunction::SplitN.into_py(py), n).to_object(py) + (PyStringFunction::SplitExact, n, inclusive).into_py_any(py) }, + StringFunction::SplitN(n) => (PyStringFunction::SplitN, n).into_py_any(py), StringFunction::Strptime(_, options) => ( - PyStringFunction::Strptime.into_py(py), - options - .format - .as_ref() - .map_or_else(|| py.None(), |s| s.to_object(py)), + PyStringFunction::Strptime, + options.format.as_ref().map(|s| s.as_str()), options.strict, options.exact, options.cache, ) - .to_object(py), + .into_py_any(py), StringFunction::Split(inclusive) => { - (PyStringFunction::Split.into_py(py), inclusive).to_object(py) + (PyStringFunction::Split, inclusive).into_py_any(py) }, StringFunction::ToDecimal(inference_length) => { - (PyStringFunction::ToDecimal.into_py(py), inference_length).to_object(py) + (PyStringFunction::ToDecimal, inference_length).into_py_any(py) }, #[cfg(feature = "nightly")] - StringFunction::Titlecase => { - (PyStringFunction::Titlecase.into_py(py),).to_object(py) - }, - StringFunction::Uppercase => { - (PyStringFunction::Uppercase.into_py(py),).to_object(py) - }, - StringFunction::ZFill => (PyStringFunction::ZFill.into_py(py),).to_object(py), + StringFunction::Titlecase => (PyStringFunction::Titlecase,).into_py_any(py), + StringFunction::Uppercase => (PyStringFunction::Uppercase,).into_py_any(py), + StringFunction::ZFill => (PyStringFunction::ZFill,).into_py_any(py), #[cfg(feature = "find_many")] StringFunction::ContainsMany { ascii_case_insensitive, - } => ( - PyStringFunction::ContainsMany.into_py(py), - ascii_case_insensitive, - ) - .to_object(py), + } => (PyStringFunction::ContainsMany, ascii_case_insensitive).into_py_any(py), #[cfg(feature = "find_many")] StringFunction::ReplaceMany { ascii_case_insensitive, - } => ( - PyStringFunction::ReplaceMany.into_py(py), - ascii_case_insensitive, - ) - .to_object(py), + } => (PyStringFunction::ReplaceMany, ascii_case_insensitive).into_py_any(py), #[cfg(feature = "find_many")] StringFunction::ExtractMany { .. } => { return Err(PyNotImplementedError::new_err("extract_many")) @@ -977,92 +935,104 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { return Err(PyNotImplementedError::new_err("find_many")) }, #[cfg(feature = "regex")] - StringFunction::EscapeRegex => { - (PyStringFunction::EscapeRegex.into_py(py),).to_object(py) - }, + StringFunction::EscapeRegex => (PyStringFunction::EscapeRegex,).into_py_any(py), }, FunctionExpr::StructExpr(_) => { return Err(PyNotImplementedError::new_err("struct expr")) }, FunctionExpr::TemporalExpr(fun) => match fun { - TemporalFunction::Millennium => (PyTemporalFunction::Millennium,).into_py(py), - TemporalFunction::Century => (PyTemporalFunction::Century,).into_py(py), - TemporalFunction::Year => (PyTemporalFunction::Year,).into_py(py), - TemporalFunction::IsLeapYear => (PyTemporalFunction::IsLeapYear,).into_py(py), - TemporalFunction::IsoYear => (PyTemporalFunction::IsoYear,).into_py(py), - TemporalFunction::Quarter => (PyTemporalFunction::Quarter,).into_py(py), - TemporalFunction::Month => (PyTemporalFunction::Month,).into_py(py), - TemporalFunction::Week => (PyTemporalFunction::Week,).into_py(py), - TemporalFunction::WeekDay => (PyTemporalFunction::WeekDay,).into_py(py), - TemporalFunction::Day => (PyTemporalFunction::Day,).into_py(py), - TemporalFunction::OrdinalDay => (PyTemporalFunction::OrdinalDay,).into_py(py), - TemporalFunction::Time => (PyTemporalFunction::Time,).into_py(py), - TemporalFunction::Date => (PyTemporalFunction::Date,).into_py(py), - TemporalFunction::Datetime => (PyTemporalFunction::Datetime,).into_py(py), + TemporalFunction::Millennium => { + (PyTemporalFunction::Millennium,).into_py_any(py) + }, + TemporalFunction::Century => (PyTemporalFunction::Century,).into_py_any(py), + TemporalFunction::Year => (PyTemporalFunction::Year,).into_py_any(py), + TemporalFunction::IsLeapYear => { + (PyTemporalFunction::IsLeapYear,).into_py_any(py) + }, + TemporalFunction::IsoYear => (PyTemporalFunction::IsoYear,).into_py_any(py), + TemporalFunction::Quarter => (PyTemporalFunction::Quarter,).into_py_any(py), + TemporalFunction::Month => (PyTemporalFunction::Month,).into_py_any(py), + TemporalFunction::Week => (PyTemporalFunction::Week,).into_py_any(py), + TemporalFunction::WeekDay => (PyTemporalFunction::WeekDay,).into_py_any(py), + TemporalFunction::Day => (PyTemporalFunction::Day,).into_py_any(py), + TemporalFunction::OrdinalDay => { + (PyTemporalFunction::OrdinalDay,).into_py_any(py) + }, + TemporalFunction::Time => (PyTemporalFunction::Time,).into_py_any(py), + TemporalFunction::Date => (PyTemporalFunction::Date,).into_py_any(py), + TemporalFunction::Datetime => (PyTemporalFunction::Datetime,).into_py_any(py), TemporalFunction::Duration(time_unit) => { - (PyTemporalFunction::Duration, Wrap(*time_unit)).into_py(py) - }, - TemporalFunction::Hour => (PyTemporalFunction::Hour,).into_py(py), - TemporalFunction::Minute => (PyTemporalFunction::Minute,).into_py(py), - TemporalFunction::Second => (PyTemporalFunction::Second,).into_py(py), - TemporalFunction::Millisecond => (PyTemporalFunction::Millisecond,).into_py(py), - TemporalFunction::Microsecond => (PyTemporalFunction::Microsecond,).into_py(py), - TemporalFunction::Nanosecond => (PyTemporalFunction::Nanosecond,).into_py(py), - TemporalFunction::TotalDays => (PyTemporalFunction::TotalDays,).into_py(py), - TemporalFunction::TotalHours => (PyTemporalFunction::TotalHours,).into_py(py), + (PyTemporalFunction::Duration, Wrap(*time_unit)).into_py_any(py) + }, + TemporalFunction::Hour => (PyTemporalFunction::Hour,).into_py_any(py), + TemporalFunction::Minute => (PyTemporalFunction::Minute,).into_py_any(py), + TemporalFunction::Second => (PyTemporalFunction::Second,).into_py_any(py), + TemporalFunction::Millisecond => { + (PyTemporalFunction::Millisecond,).into_py_any(py) + }, + TemporalFunction::Microsecond => { + (PyTemporalFunction::Microsecond,).into_py_any(py) + }, + TemporalFunction::Nanosecond => { + (PyTemporalFunction::Nanosecond,).into_py_any(py) + }, + TemporalFunction::TotalDays => (PyTemporalFunction::TotalDays,).into_py_any(py), + TemporalFunction::TotalHours => { + (PyTemporalFunction::TotalHours,).into_py_any(py) + }, TemporalFunction::TotalMinutes => { - (PyTemporalFunction::TotalMinutes,).into_py(py) + (PyTemporalFunction::TotalMinutes,).into_py_any(py) }, TemporalFunction::TotalSeconds => { - (PyTemporalFunction::TotalSeconds,).into_py(py) + (PyTemporalFunction::TotalSeconds,).into_py_any(py) }, TemporalFunction::TotalMilliseconds => { - (PyTemporalFunction::TotalMilliseconds,).into_py(py) + (PyTemporalFunction::TotalMilliseconds,).into_py_any(py) }, TemporalFunction::TotalMicroseconds => { - (PyTemporalFunction::TotalMicroseconds,).into_py(py) + (PyTemporalFunction::TotalMicroseconds,).into_py_any(py) }, TemporalFunction::TotalNanoseconds => { - (PyTemporalFunction::TotalNanoseconds,).into_py(py) + (PyTemporalFunction::TotalNanoseconds,).into_py_any(py) }, TemporalFunction::ToString(format) => { - (PyTemporalFunction::ToString, format).into_py(py) + (PyTemporalFunction::ToString, format).into_py_any(py) }, TemporalFunction::CastTimeUnit(time_unit) => { - (PyTemporalFunction::CastTimeUnit, Wrap(*time_unit)).into_py(py) + (PyTemporalFunction::CastTimeUnit, Wrap(*time_unit)).into_py_any(py) }, TemporalFunction::WithTimeUnit(time_unit) => { - (PyTemporalFunction::WithTimeUnit, Wrap(*time_unit)).into_py(py) + (PyTemporalFunction::WithTimeUnit, Wrap(*time_unit)).into_py_any(py) }, #[cfg(feature = "timezones")] TemporalFunction::ConvertTimeZone(time_zone) => { - (PyTemporalFunction::ConvertTimeZone, time_zone.as_str()).into_py(py) + (PyTemporalFunction::ConvertTimeZone, time_zone.as_str()).into_py_any(py) }, TemporalFunction::TimeStamp(time_unit) => { - (PyTemporalFunction::TimeStamp, Wrap(*time_unit)).into_py(py) + (PyTemporalFunction::TimeStamp, Wrap(*time_unit)).into_py_any(py) + }, + TemporalFunction::Truncate => (PyTemporalFunction::Truncate,).into_py_any(py), + TemporalFunction::OffsetBy => (PyTemporalFunction::OffsetBy,).into_py_any(py), + TemporalFunction::MonthStart => { + (PyTemporalFunction::MonthStart,).into_py_any(py) }, - TemporalFunction::Truncate => (PyTemporalFunction::Truncate,).into_py(py), - TemporalFunction::OffsetBy => (PyTemporalFunction::OffsetBy,).into_py(py), - TemporalFunction::MonthStart => (PyTemporalFunction::MonthStart,).into_py(py), - TemporalFunction::MonthEnd => (PyTemporalFunction::MonthEnd,).into_py(py), + TemporalFunction::MonthEnd => (PyTemporalFunction::MonthEnd,).into_py_any(py), #[cfg(feature = "timezones")] TemporalFunction::BaseUtcOffset => { - (PyTemporalFunction::BaseUtcOffset,).into_py(py) + (PyTemporalFunction::BaseUtcOffset,).into_py_any(py) }, #[cfg(feature = "timezones")] - TemporalFunction::DSTOffset => (PyTemporalFunction::DSTOffset,).into_py(py), - TemporalFunction::Round => (PyTemporalFunction::Round,).into_py(py), + TemporalFunction::DSTOffset => (PyTemporalFunction::DSTOffset,).into_py_any(py), + TemporalFunction::Round => (PyTemporalFunction::Round,).into_py_any(py), #[cfg(feature = "timezones")] TemporalFunction::ReplaceTimeZone(time_zone, non_existent) => ( PyTemporalFunction::ReplaceTimeZone, - time_zone - .as_ref() - .map_or_else(|| py.None(), |s| s.to_object(py)), + time_zone.as_ref().map(|s| s.as_str()), Into::<&str>::into(non_existent), ) - .into_py(py), + .into_py_any(py), TemporalFunction::Combine(time_unit) => { - (PyTemporalFunction::Combine, Wrap(*time_unit)).into_py(py) + (PyTemporalFunction::Combine, Wrap(*time_unit)).into_py_any(py) }, TemporalFunction::DatetimeFunction { time_unit, @@ -1070,63 +1040,63 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { } => ( PyTemporalFunction::DatetimeFunction, Wrap(*time_unit), - time_zone - .as_ref() - .map_or_else(|| py.None(), |s| s.to_object(py)), + time_zone.as_ref().map(|s| s.as_str()), ) - .into_py(py), + .into_py_any(py), }, FunctionExpr::Boolean(boolfun) => match boolfun { BooleanFunction::Any { ignore_nulls } => { - (PyBooleanFunction::Any, *ignore_nulls).into_py(py) + (PyBooleanFunction::Any, *ignore_nulls).into_py_any(py) }, BooleanFunction::All { ignore_nulls } => { - (PyBooleanFunction::All, *ignore_nulls).into_py(py) - }, - BooleanFunction::IsNull => (PyBooleanFunction::IsNull,).into_py(py), - BooleanFunction::IsNotNull => (PyBooleanFunction::IsNotNull,).into_py(py), - BooleanFunction::IsFinite => (PyBooleanFunction::IsFinite,).into_py(py), - BooleanFunction::IsInfinite => (PyBooleanFunction::IsInfinite,).into_py(py), - BooleanFunction::IsNan => (PyBooleanFunction::IsNan,).into_py(py), - BooleanFunction::IsNotNan => (PyBooleanFunction::IsNotNan,).into_py(py), + (PyBooleanFunction::All, *ignore_nulls).into_py_any(py) + }, + BooleanFunction::IsNull => (PyBooleanFunction::IsNull,).into_py_any(py), + BooleanFunction::IsNotNull => (PyBooleanFunction::IsNotNull,).into_py_any(py), + BooleanFunction::IsFinite => (PyBooleanFunction::IsFinite,).into_py_any(py), + BooleanFunction::IsInfinite => (PyBooleanFunction::IsInfinite,).into_py_any(py), + BooleanFunction::IsNan => (PyBooleanFunction::IsNan,).into_py_any(py), + BooleanFunction::IsNotNan => (PyBooleanFunction::IsNotNan,).into_py_any(py), BooleanFunction::IsFirstDistinct => { - (PyBooleanFunction::IsFirstDistinct,).into_py(py) + (PyBooleanFunction::IsFirstDistinct,).into_py_any(py) }, BooleanFunction::IsLastDistinct => { - (PyBooleanFunction::IsLastDistinct,).into_py(py) + (PyBooleanFunction::IsLastDistinct,).into_py_any(py) + }, + BooleanFunction::IsUnique => (PyBooleanFunction::IsUnique,).into_py_any(py), + BooleanFunction::IsDuplicated => { + (PyBooleanFunction::IsDuplicated,).into_py_any(py) }, - BooleanFunction::IsUnique => (PyBooleanFunction::IsUnique,).into_py(py), - BooleanFunction::IsDuplicated => (PyBooleanFunction::IsDuplicated,).into_py(py), BooleanFunction::IsBetween { closed } => { - (PyBooleanFunction::IsBetween, Into::<&str>::into(closed)).into_py(py) + (PyBooleanFunction::IsBetween, Into::<&str>::into(closed)).into_py_any(py) }, #[cfg(feature = "is_in")] - BooleanFunction::IsIn => (PyBooleanFunction::IsIn,).into_py(py), + BooleanFunction::IsIn => (PyBooleanFunction::IsIn,).into_py_any(py), BooleanFunction::AllHorizontal => { - (PyBooleanFunction::AllHorizontal,).into_py(py) + (PyBooleanFunction::AllHorizontal,).into_py_any(py) }, BooleanFunction::AnyHorizontal => { - (PyBooleanFunction::AnyHorizontal,).into_py(py) + (PyBooleanFunction::AnyHorizontal,).into_py_any(py) }, - BooleanFunction::Not => (PyBooleanFunction::Not,).into_py(py), + BooleanFunction::Not => (PyBooleanFunction::Not,).into_py_any(py), }, - FunctionExpr::Abs => ("abs",).to_object(py), + FunctionExpr::Abs => ("abs",).into_py_any(py), #[cfg(feature = "hist")] FunctionExpr::Hist { bin_count, include_category, include_breakpoint, - } => ("hist", bin_count, include_category, include_breakpoint).to_object(py), - FunctionExpr::NullCount => ("null_count",).to_object(py), + } => ("hist", bin_count, include_category, include_breakpoint).into_py_any(py), + FunctionExpr::NullCount => ("null_count",).into_py_any(py), FunctionExpr::Pow(f) => match f { - PowFunction::Generic => ("pow",).to_object(py), - PowFunction::Sqrt => ("sqrt",).to_object(py), - PowFunction::Cbrt => ("cbrt",).to_object(py), + PowFunction::Generic => ("pow",).into_py_any(py), + PowFunction::Sqrt => ("sqrt",).into_py_any(py), + PowFunction::Cbrt => ("cbrt",).into_py_any(py), }, FunctionExpr::Hash(seed, seed_1, seed_2, seed_3) => { - ("hash", seed, seed_1, seed_2, seed_3).to_object(py) + ("hash", seed, seed_1, seed_2, seed_3).into_py_any(py) }, - FunctionExpr::ArgWhere => ("argwhere",).to_object(py), + FunctionExpr::ArgWhere => ("argwhere",).into_py_any(py), #[cfg(feature = "search_sorted")] FunctionExpr::SearchSorted(side) => ( "search_sorted", @@ -1136,7 +1106,7 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { SearchSortedSide::Right => "right", }, ) - .to_object(py), + .into_py_any(py), FunctionExpr::Range(_) => return Err(PyNotImplementedError::new_err("range")), #[cfg(feature = "trigonometry")] FunctionExpr::Trigonometry(trigfun) => { @@ -1159,13 +1129,13 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { TrigonometricFunction::Degrees => ("degrees",), TrigonometricFunction::Radians => ("radians",), } - .to_object(py) + .into_py_any(py) }, #[cfg(feature = "trigonometry")] - FunctionExpr::Atan2 => ("atan2",).to_object(py), + FunctionExpr::Atan2 => ("atan2",).into_py_any(py), #[cfg(feature = "sign")] - FunctionExpr::Sign => ("sign",).to_object(py), - FunctionExpr::FillNull => ("fill_null",).to_object(py), + FunctionExpr::Sign => ("sign",).into_py_any(py), + FunctionExpr::FillNull => ("fill_null",).into_py_any(py), FunctionExpr::RollingExpr(rolling) => match rolling { RollingFunction::Min(_) => { return Err(PyNotImplementedError::new_err("rolling min")) @@ -1218,42 +1188,44 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { return Err(PyNotImplementedError::new_err("rolling std by")) }, }, - FunctionExpr::ShiftAndFill => ("shift_and_fill",).to_object(py), - FunctionExpr::Shift => ("shift",).to_object(py), - FunctionExpr::DropNans => ("drop_nans",).to_object(py), - FunctionExpr::DropNulls => ("drop_nulls",).to_object(py), - FunctionExpr::Mode => ("mode",).to_object(py), - FunctionExpr::Skew(bias) => ("skew", bias).to_object(py), - FunctionExpr::Kurtosis(fisher, bias) => ("kurtosis", fisher, bias).to_object(py), + FunctionExpr::ShiftAndFill => ("shift_and_fill",).into_py_any(py), + FunctionExpr::Shift => ("shift",).into_py_any(py), + FunctionExpr::DropNans => ("drop_nans",).into_py_any(py), + FunctionExpr::DropNulls => ("drop_nulls",).into_py_any(py), + FunctionExpr::Mode => ("mode",).into_py_any(py), + FunctionExpr::Skew(bias) => ("skew", bias).into_py_any(py), + FunctionExpr::Kurtosis(fisher, bias) => ("kurtosis", fisher, bias).into_py_any(py), FunctionExpr::Reshape(_) => return Err(PyNotImplementedError::new_err("reshape")), #[cfg(feature = "repeat_by")] - FunctionExpr::RepeatBy => ("repeat_by",).to_object(py), - FunctionExpr::ArgUnique => ("arg_unique",).to_object(py), - FunctionExpr::Repeat => ("repeat",).to_object(py), + FunctionExpr::RepeatBy => ("repeat_by",).into_py_any(py), + FunctionExpr::ArgUnique => ("arg_unique",).into_py_any(py), + FunctionExpr::Repeat => ("repeat",).into_py_any(py), FunctionExpr::Rank { options: _, seed: _, } => return Err(PyNotImplementedError::new_err("rank")), - FunctionExpr::Clip { has_min, has_max } => ("clip", has_min, has_max).to_object(py), - FunctionExpr::AsStruct => ("as_struct",).to_object(py), + FunctionExpr::Clip { has_min, has_max } => { + ("clip", has_min, has_max).into_py_any(py) + }, + FunctionExpr::AsStruct => ("as_struct",).into_py_any(py), #[cfg(feature = "top_k")] - FunctionExpr::TopK { descending } => ("top_k", descending).to_object(py), - FunctionExpr::CumCount { reverse } => ("cum_count", reverse).to_object(py), - FunctionExpr::CumSum { reverse } => ("cum_sum", reverse).to_object(py), - FunctionExpr::CumProd { reverse } => ("cum_prod", reverse).to_object(py), - FunctionExpr::CumMin { reverse } => ("cum_min", reverse).to_object(py), - FunctionExpr::CumMax { reverse } => ("cum_max", reverse).to_object(py), - FunctionExpr::Reverse => ("reverse",).to_object(py), + FunctionExpr::TopK { descending } => ("top_k", descending).into_py_any(py), + FunctionExpr::CumCount { reverse } => ("cum_count", reverse).into_py_any(py), + FunctionExpr::CumSum { reverse } => ("cum_sum", reverse).into_py_any(py), + FunctionExpr::CumProd { reverse } => ("cum_prod", reverse).into_py_any(py), + FunctionExpr::CumMin { reverse } => ("cum_min", reverse).into_py_any(py), + FunctionExpr::CumMax { reverse } => ("cum_max", reverse).into_py_any(py), + FunctionExpr::Reverse => ("reverse",).into_py_any(py), FunctionExpr::ValueCounts { sort, parallel, name, normalize, - } => ("value_counts", sort, parallel, name.as_str(), normalize).to_object(py), - FunctionExpr::UniqueCounts => ("unique_counts",).to_object(py), - FunctionExpr::ApproxNUnique => ("approx_n_unique",).to_object(py), - FunctionExpr::Coalesce => ("coalesce",).to_object(py), - FunctionExpr::ShrinkType => ("shrink_dtype",).to_object(py), + } => ("value_counts", sort, parallel, name.as_str(), normalize).into_py_any(py), + FunctionExpr::UniqueCounts => ("unique_counts",).into_py_any(py), + FunctionExpr::ApproxNUnique => ("approx_n_unique",).into_py_any(py), + FunctionExpr::Coalesce => ("coalesce",).into_py_any(py), + FunctionExpr::ShrinkType => ("shrink_dtype",).into_py_any(py), FunctionExpr::Diff(n, null_behaviour) => ( "diff", n, @@ -1262,9 +1234,9 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { NullBehavior::Ignore => "ignore", }, ) - .to_object(py), + .into_py_any(py), #[cfg(feature = "pct_change")] - FunctionExpr::PctChange => ("pct_change",).to_object(py), + FunctionExpr::PctChange => ("pct_change",).into_py_any(py), FunctionExpr::Interpolate(method) => ( "interpolate", match method { @@ -1272,21 +1244,21 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { InterpolationMethod::Nearest => "nearest", }, ) - .to_object(py), - FunctionExpr::InterpolateBy => ("interpolate_by",).to_object(py), + .into_py_any(py), + FunctionExpr::InterpolateBy => ("interpolate_by",).into_py_any(py), FunctionExpr::Entropy { base, normalize } => { - ("entropy", base, normalize).to_object(py) + ("entropy", base, normalize).into_py_any(py) }, - FunctionExpr::Log { base } => ("log", base).to_object(py), - FunctionExpr::Log1p => ("log1p",).to_object(py), - FunctionExpr::Exp => ("exp",).to_object(py), - FunctionExpr::Unique(maintain_order) => ("unique", maintain_order).to_object(py), - FunctionExpr::Round { decimals } => ("round", decimals).to_object(py), - FunctionExpr::RoundSF { digits } => ("round_sig_figs", digits).to_object(py), - FunctionExpr::Floor => ("floor",).to_object(py), - FunctionExpr::Ceil => ("ceil",).to_object(py), - FunctionExpr::UpperBound => ("upper_bound",).to_object(py), - FunctionExpr::LowerBound => ("lower_bound",).to_object(py), + FunctionExpr::Log { base } => ("log", base).into_py_any(py), + FunctionExpr::Log1p => ("log1p",).into_py_any(py), + FunctionExpr::Exp => ("exp",).into_py_any(py), + FunctionExpr::Unique(maintain_order) => ("unique", maintain_order).into_py_any(py), + FunctionExpr::Round { decimals } => ("round", decimals).into_py_any(py), + FunctionExpr::RoundSF { digits } => ("round_sig_figs", digits).into_py_any(py), + FunctionExpr::Floor => ("floor",).into_py_any(py), + FunctionExpr::Ceil => ("ceil",).into_py_any(py), + FunctionExpr::UpperBound => ("upper_bound",).into_py_any(py), + FunctionExpr::LowerBound => ("lower_bound",).into_py_any(py), FunctionExpr::Fused(_) => return Err(PyNotImplementedError::new_err("fused")), FunctionExpr::ConcatExpr(_) => { return Err(PyNotImplementedError::new_err("concat expr")) @@ -1295,18 +1267,18 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { return Err(PyNotImplementedError::new_err("corr")) }, #[cfg(feature = "peaks")] - FunctionExpr::PeakMin => ("peak_max",).to_object(py), + FunctionExpr::PeakMin => ("peak_max",).into_py_any(py), #[cfg(feature = "peaks")] - FunctionExpr::PeakMax => ("peak_min",).to_object(py), + FunctionExpr::PeakMax => ("peak_min",).into_py_any(py), #[cfg(feature = "cutqcut")] FunctionExpr::Cut { .. } => return Err(PyNotImplementedError::new_err("cut")), #[cfg(feature = "cutqcut")] FunctionExpr::QCut { .. } => return Err(PyNotImplementedError::new_err("qcut")), #[cfg(feature = "rle")] - FunctionExpr::RLE => ("rle",).to_object(py), + FunctionExpr::RLE => ("rle",).into_py_any(py), #[cfg(feature = "rle")] - FunctionExpr::RLEID => ("rle_id",).to_object(py), - FunctionExpr::ToPhysical => ("to_physical",).to_object(py), + FunctionExpr::RLEID => ("rle_id",).into_py_any(py), + FunctionExpr::ToPhysical => ("to_physical",).into_py_any(py), FunctionExpr::Random { .. } => { return Err(PyNotImplementedError::new_err("random")) }, @@ -1318,21 +1290,21 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { IsSorted::Not => "not", }, ) - .to_object(py), + .into_py_any(py), #[cfg(feature = "ffi_plugin")] FunctionExpr::FfiPlugin { .. } => { return Err(PyNotImplementedError::new_err("ffi plugin")) }, - FunctionExpr::BackwardFill { limit } => ("backward_fill", limit).to_object(py), - FunctionExpr::ForwardFill { limit } => ("forward_fill", limit).to_object(py), + FunctionExpr::BackwardFill { limit } => ("backward_fill", limit).into_py_any(py), + FunctionExpr::ForwardFill { limit } => ("forward_fill", limit).into_py_any(py), FunctionExpr::SumHorizontal { ignore_nulls } => { - ("sum_horizontal", ignore_nulls).to_object(py) + ("sum_horizontal", ignore_nulls).into_py_any(py) }, - FunctionExpr::MaxHorizontal => ("max_horizontal",).to_object(py), + FunctionExpr::MaxHorizontal => ("max_horizontal",).into_py_any(py), FunctionExpr::MeanHorizontal { ignore_nulls } => { - ("mean_horizontal", ignore_nulls).to_object(py) + ("mean_horizontal", ignore_nulls).into_py_any(py) }, - FunctionExpr::MinHorizontal => ("min_horizontal",).to_object(py), + FunctionExpr::MinHorizontal => ("min_horizontal",).into_py_any(py), FunctionExpr::EwmMean { options: _ } => { return Err(PyNotImplementedError::new_err("ewm mean")) }, @@ -1342,32 +1314,32 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { FunctionExpr::EwmVar { options: _ } => { return Err(PyNotImplementedError::new_err("ewm var")) }, - FunctionExpr::Replace => ("replace",).to_object(py), + FunctionExpr::Replace => ("replace",).into_py_any(py), FunctionExpr::ReplaceStrict { return_dtype: _ } => { // Can ignore the return dtype because it is encoded in the schema. - ("replace_strict",).to_object(py) + ("replace_strict",).into_py_any(py) }, - FunctionExpr::Negate => ("negate",).to_object(py), + FunctionExpr::Negate => ("negate",).into_py_any(py), FunctionExpr::FillNullWithStrategy(_) => { return Err(PyNotImplementedError::new_err("fill null with strategy")) }, FunctionExpr::GatherEvery { n, offset } => { - ("gather_every", offset, n).to_object(py) + ("gather_every", offset, n).into_py_any(py) }, - FunctionExpr::Reinterpret(signed) => ("reinterpret", signed).to_object(py), - FunctionExpr::ExtendConstant => ("extend_constant",).to_object(py), + FunctionExpr::Reinterpret(signed) => ("reinterpret", signed).into_py_any(py), + FunctionExpr::ExtendConstant => ("extend_constant",).into_py_any(py), FunctionExpr::Business(_) => { return Err(PyNotImplementedError::new_err("business")) }, #[cfg(feature = "top_k")] - FunctionExpr::TopKBy { descending } => ("top_k_by", descending).to_object(py), + FunctionExpr::TopKBy { descending } => ("top_k_by", descending).into_py_any(py), FunctionExpr::EwmMeanBy { half_life: _ } => { return Err(PyNotImplementedError::new_err("ewm_mean_by")) }, - }, + }?, options: py.None(), } - .into_py(py), + .into_py_any(py), AExpr::Window { function, partition_by, @@ -1385,11 +1357,11 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { let order_by = order_by.map(|(n, _)| n.0); let options = match options { - WindowType::Over(options) => PyWindowMapping { inner: *options }.into_py(py), + WindowType::Over(options) => PyWindowMapping { inner: *options }.into_py_any(py)?, WindowType::Rolling(options) => PyRollingGroupOptions { inner: options.clone(), } - .into_py(py), + .into_py_any(py)?, }; Window { function, @@ -1399,7 +1371,7 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { order_by_nulls_last, options, } - .into_py(py) + .into_py_any(py) }, AExpr::Slice { input, @@ -1410,8 +1382,7 @@ pub(crate) fn into_py(py: Python<'_>, expr: &AExpr) -> PyResult { offset: offset.0, length: length.0, } - .into_py(py), - AExpr::Len => Len {}.into_py(py), - }; - Ok(result) + .into_py_any(py), + AExpr::Len => Len {}.into_py_any(py), + } } diff --git a/crates/polars-python/src/lazyframe/visitor/nodes.rs b/crates/polars-python/src/lazyframe/visitor/nodes.rs index ccb666e6dcba..5fb2e900a745 100644 --- a/crates/polars-python/src/lazyframe/visitor/nodes.rs +++ b/crates/polars-python/src/lazyframe/visitor/nodes.rs @@ -6,6 +6,7 @@ use polars_plan::prelude::{ }; use pyo3::exceptions::{PyNotImplementedError, PyValueError}; use pyo3::prelude::*; +use pyo3::IntoPyObjectExt; use super::expr_nodes::PyGroupbyOptions; use crate::lazyframe::visit::PyExprIR; @@ -47,43 +48,37 @@ pub struct PyFileOptions { #[pymethods] impl PyFileOptions { #[getter] - fn n_rows(&self, py: Python<'_>) -> PyResult { - Ok(self - .inner - .slice - .map_or_else(|| py.None(), |n| n.into_py(py))) + fn n_rows(&self) -> Option<(i64, usize)> { + self.inner.slice } #[getter] - fn with_columns(&self, py: Python<'_>) -> PyResult { - Ok(self.inner.with_columns.as_ref().map_or_else( - || py.None(), - |cols| { - cols.iter() - .map(|x| x.as_str()) - .collect::>() - .to_object(py) - }, - )) + fn with_columns(&self) -> Option> { + self.inner + .with_columns + .as_ref()? + .iter() + .map(|x| x.as_str()) + .collect::>() + .into() } #[getter] - fn cache(&self, _py: Python<'_>) -> PyResult { - Ok(self.inner.cache) + fn cache(&self, _py: Python<'_>) -> bool { + self.inner.cache } #[getter] - fn row_index(&self, py: Python<'_>) -> PyResult { - Ok(self - .inner + fn row_index(&self) -> Option<(&str, IdxSize)> { + self.inner .row_index .as_ref() - .map_or_else(|| py.None(), |n| (n.name.as_str(), n.offset).to_object(py))) + .map(|n| (n.name.as_str(), n.offset)) } #[getter] - fn rechunk(&self, _py: Python<'_>) -> PyResult { - Ok(self.inner.rechunk) + fn rechunk(&self, _py: Python<'_>) -> bool { + self.inner.rechunk } #[getter] - fn file_counter(&self, _py: Python<'_>) -> PyResult { - Ok(self.inner.file_counter) + fn file_counter(&self, _py: Python<'_>) -> FileCount { + self.inner.file_counter } #[getter] fn hive_options(&self, _py: Python<'_>) -> PyResult { @@ -260,7 +255,7 @@ pub struct Sink { } pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { - let result = match plan { + match plan { IR::PythonScan { options } => { let python_src = match options.python_source { PythonScanSource::Pyarrow => "pyarrow", @@ -275,47 +270,45 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { .as_ref() .map_or_else(|| py.None(), |s| s.0.clone_ref(py)), options.with_columns.as_ref().map_or_else( - || py.None(), + || Ok(py.None()), |cols| { cols.iter() .map(|x| x.as_str()) .collect::>() - .to_object(py) + .into_py_any(py) }, - ), + )?, python_src, match &options.predicate { PythonPredicate::None => py.None(), - PythonPredicate::PyArrow(s) => ("pyarrow", s).to_object(py), - PythonPredicate::Polars(e) => ("polars", e.node().0).to_object(py), + PythonPredicate::PyArrow(s) => ("pyarrow", s).into_py_any(py)?, + PythonPredicate::Polars(e) => ("polars", e.node().0).into_py_any(py)?, }, options .n_rows - .map_or_else(|| py.None(), |s| s.to_object(py)), + .map_or_else(|| Ok(py.None()), |s| s.into_py_any(py))?, ) - .to_object(py), + .into_py_any(py)?, } - .into_py(py) + .into_py_any(py) }, IR::Slice { input, offset, len } => Slice { input: input.0, offset: *offset, len: *len, } - .into_py(py), + .into_py_any(py), IR::Filter { input, predicate } => Filter { input: input.0, predicate: predicate.into(), } - .into_py(py), + .into_py_any(py), IR::Scan { hive_parts: Some(_), .. - } => { - return Err(PyNotImplementedError::new_err( - "scan with hive partitioning", - )) - }, + } => Err(PyNotImplementedError::new_err( + "scan with hive partitioning", + )), IR::Scan { sources, file_info: _, @@ -328,7 +321,7 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { paths: sources .into_paths() .ok_or_else(|| PyNotImplementedError::new_err("scan with BytesIO"))? - .to_object(py), + .into_py_any(py)?, // TODO: file info file_info: py.None(), predicate: predicate.as_ref().map(|e| e.into()), @@ -347,7 +340,7 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { .map_err(|err| PyValueError::new_err(format!("{err:?}")))?; let cloud_options = serde_json::to_string(cloud_options) .map_err(|err| PyValueError::new_err(format!("{err:?}")))?; - ("csv", options, cloud_options).into_py(py) + ("csv", options, cloud_options).into_py_any(py)? }, #[cfg(feature = "parquet")] FileScan::Parquet { @@ -359,7 +352,7 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { .map_err(|err| PyValueError::new_err(format!("{err:?}")))?; let cloud_options = serde_json::to_string(cloud_options) .map_err(|err| PyValueError::new_err(format!("{err:?}")))?; - ("parquet", options, cloud_options).into_py(py) + ("parquet", options, cloud_options).into_py_any(py)? }, #[cfg(feature = "ipc")] FileScan::Ipc { .. } => return Err(PyNotImplementedError::new_err("ipc scan")), @@ -368,14 +361,14 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { // TODO: Also pass cloud_options let options = serde_json::to_string(options) .map_err(|err| PyValueError::new_err(format!("{err:?}")))?; - ("ndjson", options).into_py(py) + ("ndjson", options).into_py_any(py)? }, FileScan::Anonymous { .. } => { return Err(PyNotImplementedError::new_err("anonymous scan")) }, }, } - .into_py(py), + .into_py_any(py), IR::DataFrameScan { df, schema: _, @@ -384,19 +377,19 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { } => DataFrameScan { df: PyDataFrame::new((**df).clone()), projection: output_schema.as_ref().map_or_else( - || py.None(), + || Ok(py.None()), |s| { s.iter_names() .map(|s| s.as_str()) .collect::>() - .to_object(py) + .into_py_any(py) }, - ), + )?, selection: selection.as_ref().map(|e| e.into()), } - .into_py(py), + .into_py_any(py), IR::SimpleProjection { input, columns: _ } => { - SimpleProjection { input: input.0 }.into_py(py) + SimpleProjection { input: input.0 }.into_py_any(py) }, IR::Select { input, @@ -408,7 +401,7 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { input: input.0, should_broadcast: options.should_broadcast, } - .into_py(py), + .into_py_any(py), IR::Sort { input, by_column, @@ -424,7 +417,7 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { ), slice: *slice, } - .into_py(py), + .into_py_any(py), IR::Cache { input, id, @@ -434,7 +427,7 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { id_: *id, cache_hits: *cache_hits, } - .into_py(py), + .into_py_any(py), IR::GroupBy { input, keys, @@ -454,9 +447,9 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { ))) })?, maintain_order: *maintain_order, - options: PyGroupbyOptions::new(options.as_ref().clone()).into_py(py), + options: PyGroupbyOptions::new(options.as_ref().clone()).into_py_any(py)?, } - .into_py(py), + .into_py_any(py), IR::Join { input_left, input_right, @@ -471,7 +464,7 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { right_on: right_on.iter().map(|e| e.into()).collect(), options: { let how = &options.args.how; - let name = Into::<&str>::into(how).to_object(py); + let name = Into::<&str>::into(how).into_pyobject(py)?; ( match how { #[cfg(feature = "asof_join")] @@ -481,14 +474,14 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { #[cfg(feature = "iejoin")] JoinType::IEJoin(ie_options) => ( name, - crate::Wrap(ie_options.operator1).into_py(py), - ie_options - .operator2 - .as_ref() - .map_or_else(|| py.None(), |op| crate::Wrap(*op).into_py(py)), + crate::Wrap(ie_options.operator1).into_py_any(py)?, + ie_options.operator2.as_ref().map_or_else( + || Ok(py.None()), + |op| crate::Wrap(*op).into_py_any(py), + )?, ) - .into_py(py), - _ => name, + .into_py_any(py)?, + _ => name.into_any().unbind(), }, options.args.join_nulls, options.args.slice, @@ -496,10 +489,10 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { options.args.coalesce.coalesce(how), Into::<&str>::into(options.args.maintain_order), ) - .to_object(py) + .into_py_any(py)? }, } - .into_py(py), + .into_py_any(py), IR::HStack { input, exprs, @@ -510,7 +503,7 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { exprs: exprs.iter().map(|e| e.into()).collect(), should_broadcast: options.should_broadcast, } - .into_py(py), + .into_py_any(py), IR::Reduce { input, exprs, @@ -519,26 +512,26 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { input: input.0, exprs: exprs.iter().map(|e| e.into()).collect(), } - .into_py(py), + .into_py_any(py), IR::Distinct { input, options } => Distinct { input: input.0, options: ( Into::<&str>::into(options.keep_strategy), options.subset.as_ref().map_or_else( - || py.None(), + || Ok(py.None()), |f| { f.iter() .map(|s| s.as_ref()) .collect::>() - .to_object(py) + .into_py_any(py) }, - ), + )?, options.maintain_order, options.slice, ) - .to_object(py), + .into_py_any(py)?, } - .into_py(py), + .into_py_any(py), IR::MapFunction { input, function } => MapFunction { input: input.0, function: match function { @@ -562,11 +555,11 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { "unnest", columns.iter().map(|s| s.to_string()).collect::>(), ) - .to_object(py), - FunctionIR::Rechunk => ("rechunk",).to_object(py), + .into_py_any(py)?, + FunctionIR::Rechunk => ("rechunk",).into_py_any(py)?, #[cfg(feature = "merge_sorted")] FunctionIR::MergeSorted { column } => { - ("merge_sorted", column.to_string()).to_object(py) + ("merge_sorted", column.to_string()).into_py_any(py)? }, FunctionIR::Rename { existing, @@ -579,12 +572,12 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { new.iter().map(|s| s.as_str()).collect::>(), *swapping, ) - .to_object(py), + .into_py_any(py)?, FunctionIR::Explode { columns, schema: _ } => ( "explode", columns.iter().map(|s| s.to_string()).collect::>(), ) - .to_object(py), + .into_py_any(py)?, #[cfg(feature = "pivot")] FunctionIR::Unpivot { args, schema: _ } => ( "unpivot", @@ -592,17 +585,17 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { args.on.iter().map(|s| s.as_str()).collect::>(), args.variable_name .as_ref() - .map_or_else(|| py.None(), |s| s.as_str().to_object(py)), + .map_or_else(|| Ok(py.None()), |s| s.as_str().into_py_any(py))?, args.value_name .as_ref() - .map_or_else(|| py.None(), |s| s.as_str().to_object(py)), + .map_or_else(|| Ok(py.None()), |s| s.as_str().into_py_any(py))?, ) - .to_object(py), + .into_py_any(py)?, FunctionIR::RowIndex { name, schema: _, offset, - } => ("row_index", name.to_string(), offset.unwrap_or(0)).to_object(py), + } => ("row_index", name.to_string(), offset.unwrap_or(0)).into_py_any(py)?, FunctionIR::FastCount { sources: _, scan_type: _, @@ -610,13 +603,13 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { } => return Err(PyNotImplementedError::new_err("function count")), }, } - .into_py(py), + .into_py_any(py), IR::Union { inputs, options } => Union { inputs: inputs.iter().map(|n| n.0).collect(), // TODO: rest of options options: options.slice, } - .into_py(py), + .into_py_any(py), IR::HConcat { inputs, schema: _, @@ -625,7 +618,7 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { inputs: inputs.iter().map(|n| n.0).collect(), options: (), } - .into_py(py), + .into_py_any(py), IR::ExtContext { input, contexts, @@ -634,16 +627,13 @@ pub(crate) fn into_py(py: Python<'_>, plan: &IR) -> PyResult { input: input.0, contexts: contexts.iter().map(|n| n.0).collect(), } - .into_py(py), + .into_py_any(py), IR::Sink { input: _, payload: _, - } => { - return Err(PyNotImplementedError::new_err( - "Not expecting to see a Sink node", - )) - }, - IR::Invalid => return Err(PyNotImplementedError::new_err("Invalid")), - }; - Ok(result) + } => Err(PyNotImplementedError::new_err( + "Not expecting to see a Sink node", + )), + IR::Invalid => Err(PyNotImplementedError::new_err("Invalid")), + } } diff --git a/crates/polars-python/src/lazygroupby.rs b/crates/polars-python/src/lazygroupby.rs index d2ed68f3d568..162da98ea9fc 100644 --- a/crates/polars-python/src/lazygroupby.rs +++ b/crates/polars-python/src/lazygroupby.rs @@ -7,6 +7,7 @@ use pyo3::prelude::*; use crate::conversion::Wrap; use crate::error::PyPolarsErr; use crate::expr::ToExprs; +use crate::py_modules::polars; use crate::{PyDataFrame, PyExpr, PyLazyFrame}; #[pyclass] @@ -51,7 +52,7 @@ impl PyLazyGroupBy { let function = move |df: DataFrame| { Python::with_gil(|py| { // get the pypolars module - let pypolars = PyModule::import_bound(py, "polars").unwrap(); + let pypolars = polars(py).bind(py); // create a PyDataFrame struct/object for Python let pydf = PyDataFrame::new(df); diff --git a/crates/polars-python/src/map/dataframe.rs b/crates/polars-python/src/map/dataframe.rs index 1fbfe5d9232a..3136472d30ee 100644 --- a/crates/polars-python/src/map/dataframe.rs +++ b/crates/polars-python/src/map/dataframe.rs @@ -4,6 +4,7 @@ use polars_core::series::SeriesIter; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; use pyo3::types::{PyBool, PyFloat, PyInt, PyList, PyString, PyTuple}; +use pyo3::IntoPyObjectExt; use super::*; use crate::PyDataFrame; @@ -27,7 +28,7 @@ fn get_iters_skip(df: &DataFrame, n: usize) -> Vec> // the return type is Union[PySeries, PyDataFrame] and a boolean indicating if it is a dataframe or not pub fn apply_lambda_unknown<'a>( df: &'a DataFrame, - py: Python, + py: Python<'a>, lambda: Bound<'a, PyAny>, inference_size: usize, ) -> PyResult<(PyObject, bool)> { @@ -36,7 +37,7 @@ pub fn apply_lambda_unknown<'a>( for _ in 0..df.height() { let iter = iters.iter_mut().map(|it| Wrap(it.next().unwrap())); - let arg = (PyTuple::new_bound(py, iter),); + let arg = (PyTuple::new(py, iter)?,); let out = lambda.call1(arg)?; if out.is_none() { @@ -49,7 +50,7 @@ pub fn apply_lambda_unknown<'a>( apply_lambda_with_bool_out_type(df, py, lambda, null_count, first_value) .into_series(), ) - .into_py(py), + .into_py_any(py)?, false, )); } else if out.is_instance_of::() { @@ -66,7 +67,7 @@ pub fn apply_lambda_unknown<'a>( ) .into_series(), ) - .into_py(py), + .into_py_any(py)?, false, )); } else if out.is_instance_of::() { @@ -82,7 +83,7 @@ pub fn apply_lambda_unknown<'a>( ) .into_series(), ) - .into_py(py), + .into_py_any(py)?, false, )); } else if out.is_instance_of::() { @@ -92,7 +93,7 @@ pub fn apply_lambda_unknown<'a>( apply_lambda_with_string_out_type(df, py, lambda, null_count, first_value) .into_series(), ) - .into_py(py), + .into_py_any(py)?, false, )); } else if out.hasattr("_s")? { @@ -104,7 +105,7 @@ pub fn apply_lambda_unknown<'a>( apply_lambda_with_list_out_type(df, py, lambda, null_count, Some(&series), dt)? .into_series(), ) - .into_py(py), + .into_py_any(py)?, false, )); } else if out.extract::>>().is_ok() { @@ -121,7 +122,7 @@ pub fn apply_lambda_unknown<'a>( ) .map_err(PyPolarsErr::from)?, ) - .into_py(py), + .into_py_any(py)?, true, )); } else if out.is_instance_of::() || out.is_instance_of::() { @@ -151,7 +152,7 @@ where let mut iters = get_iters_skip(df, init_null_count + skip); ((init_null_count + skip)..df.height()).map(move |_| { let iter = iters.iter_mut().map(|it| Wrap(it.next().unwrap())); - let tpl = (PyTuple::new_bound(py, iter),); + let tpl = (PyTuple::new(py, iter).unwrap(),); match lambda.call1(tpl) { Ok(val) => val.extract::().ok(), Err(e) => panic!("python function failed {e}"), @@ -169,7 +170,7 @@ pub fn apply_lambda_with_primitive_out_type<'a, D>( ) -> ChunkedArray where D: PyArrowPrimitiveType, - D::Native: ToPyObject + FromPyObject<'a>, + D::Native: IntoPyObject<'a> + FromPyObject<'a>, { let skip = usize::from(first_value.is_some()); if init_null_count == df.height() { @@ -251,7 +252,7 @@ pub fn apply_lambda_with_list_out_type<'a>( let mut iters = get_iters_skip(df, init_null_count + skip); let iter = ((init_null_count + skip)..df.height()).map(|_| { let iter = iters.iter_mut().map(|it| Wrap(it.next().unwrap())); - let tpl = (PyTuple::new_bound(py, iter),); + let tpl = (PyTuple::new(py, iter).unwrap(),); match lambda.call1(tpl) { Ok(val) => match val.getattr("_s") { Ok(val) => val.extract::().ok().map(|ps| ps.series), @@ -294,7 +295,7 @@ pub fn apply_lambda_with_rows_output<'a>( let mut iters = get_iters_skip(df, init_null_count + skip); let mut row_iter = ((init_null_count + skip)..df.height()).map(|_| { let iter = iters.iter_mut().map(|it| Wrap(it.next().unwrap())); - let tpl = (PyTuple::new_bound(py, iter),); + let tpl = (PyTuple::new(py, iter).unwrap(),); let return_val = lambda.call1(tpl).map_err(|e| polars_err!(ComputeError: format!("{e}")))?; if return_val.is_none() { diff --git a/crates/polars-python/src/map/lazy.rs b/crates/polars-python/src/map/lazy.rs index 77389b974085..13189b24f0d3 100644 --- a/crates/polars-python/src/map/lazy.rs +++ b/crates/polars-python/src/map/lazy.rs @@ -3,7 +3,7 @@ use pyo3::ffi::Py_uintptr_t; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList}; -use crate::py_modules::POLARS; +use crate::py_modules::polars; use crate::series::PySeries; use crate::{PyExpr, Wrap}; @@ -11,7 +11,7 @@ pub(crate) trait ToSeries { fn to_series( &self, py: Python, - py_polars_module: &PyObject, + py_polars_module: &Py, name: &str, ) -> PolarsResult; } @@ -20,7 +20,7 @@ impl ToSeries for PyObject { fn to_series( &self, py: Python, - py_polars_module: &PyObject, + py_polars_module: &Py, name: &str, ) -> PolarsResult { let py_pyseries = match self.getattr(py, "_s") { @@ -30,7 +30,7 @@ impl ToSeries for PyObject { let res = py_polars_module .getattr(py, "Series") .unwrap() - .call1(py, (name, PyList::new_bound(py, [self]))); + .call1(py, (name, PyList::new(py, [self]).unwrap())); match res { Ok(python_s) => python_s.getattr(py, "_s").unwrap(), @@ -52,10 +52,10 @@ impl ToSeries for PyObject { // multiple chunks Err(_) => { use polars::export::arrow::ffi; - let kwargs = PyDict::new_bound(py); + let kwargs = PyDict::new(py); kwargs.set_item("in_place", true).unwrap(); py_pyseries - .call_method_bound(py, "rechunk", (), Some(&kwargs)) + .call_method(py, "rechunk", (), Some(&kwargs)) .map_err(|e| polars_err!(ComputeError: "could not rechunk: {e}"))?; // Prepare a pointer to receive the Array struct. @@ -90,7 +90,7 @@ pub(crate) fn call_lambda_with_series( s: Series, lambda: &PyObject, ) -> PyResult { - let pypolars = POLARS.downcast_bound::(py).unwrap(); + let pypolars = polars(py).bind(py); // create a PySeries struct/object for Python let pyseries = PySeries::new(s); @@ -112,7 +112,7 @@ pub(crate) fn binary_lambda( ) -> PolarsResult> { Python::with_gil(|py| { // get the pypolars module - let pypolars = PyModule::import_bound(py, "polars").unwrap(); + let pypolars = polars(py).bind(py); // create a PySeries struct/object for Python let pyseries_a = PySeries::new(a); let pyseries_b = PySeries::new(b); @@ -134,7 +134,7 @@ pub(crate) fn binary_lambda( match lambda.call1(py, (python_series_wrapper_a, python_series_wrapper_b)) { Ok(pyobj) => pyobj, Err(e) => polars_bail!( - ComputeError: "custom python function failed: {}", e.value_bound(py), + ComputeError: "custom python function failed: {}", e.value(py), ), }; let pyseries = if let Ok(expr) = result_series_wrapper.getattr(py, "_pyexpr") { @@ -151,7 +151,7 @@ pub(crate) fn binary_lambda( let s = out.select_at_idx(0).unwrap().clone(); PySeries::new(s.take_materialized_series()) } else { - return Some(result_series_wrapper.to_series(py, &pypolars.into_py(py), "")) + return Some(result_series_wrapper.to_series(py, pypolars.as_unbound(), "")) .transpose(); }; @@ -179,9 +179,9 @@ pub(crate) fn call_lambda_with_columns_slice( py: Python, s: &[Column], lambda: &PyObject, - polars_module: &PyObject, + pypolars: &Py, ) -> PyObject { - let pypolars = polars_module.downcast_bound::(py).unwrap(); + let pypolars = pypolars.bind(py); // create a PySeries struct/object for Python let iter = s.iter().map(|s| { @@ -192,12 +192,12 @@ pub(crate) fn call_lambda_with_columns_slice( python_series_wrapper }); - let wrapped_s = PyList::new_bound(py, iter); + let wrapped_s = PyList::new(py, iter).unwrap(); // call the lambda and get a python side Series wrapper match lambda.call1(py, (wrapped_s,)) { Ok(pyobj) => pyobj, - Err(e) => panic!("python function failed: {}", e.value_bound(py)), + Err(e) => panic!("python function failed: {}", e.value(py)), } } @@ -211,7 +211,7 @@ pub fn map_mul( ) -> PyExpr { // get the pypolars module // do the import outside of the function to prevent import side effects in a hot loop. - let pypolars = PyModule::import_bound(py, "polars").unwrap().to_object(py); + let pypolars = polars(py).clone_ref(py); let function = move |s: &mut [Column]| { Python::with_gil(|py| { diff --git a/crates/polars-python/src/map/series.rs b/crates/polars-python/src/map/series.rs index 16d6212b8d1e..478c44b22f51 100644 --- a/crates/polars-python/src/map/series.rs +++ b/crates/polars-python/src/map/series.rs @@ -1,15 +1,14 @@ -use polars::prelude::*; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; use pyo3::types::{PyBool, PyCFunction, PyFloat, PyList, PyString, PyTuple}; use super::*; -use crate::py_modules::SERIES; +use crate::py_modules::{pl_series, polars}; /// Find the output type and dispatch to that implementation. fn infer_and_finish<'a, A: ApplyLambda<'a>>( applyer: &'a A, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, out: &Bound<'a, PyAny>, null_count: usize, @@ -41,14 +40,14 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>( applyer .apply_lambda_with_list_out_type( py, - lambda.to_object(py), + lambda.to_owned().unbind(), null_count, Some(&series), dt, ) .map(|ca| ca.into_series().into()) } else if out.is_instance_of::() || out.is_instance_of::() { - let series = SERIES.call1(py, (out,))?; + let series = pl_series(py).call1(py, (out,))?; let py_pyseries = series.getattr(py, "_s").unwrap(); let series = py_pyseries.extract::(py).unwrap().series; @@ -65,15 +64,16 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>( // make a new python function that is: // def new_lambda(lambda: Callable): // pl.Series(lambda(value)) - let lambda_owned = lambda.to_object(py); - let new_lambda = PyCFunction::new_closure_bound(py, None, None, move |args, _kwargs| { + let lambda_owned = lambda.to_owned().unbind(); + let new_lambda = PyCFunction::new_closure(py, None, None, move |args, _kwargs| { Python::with_gil(|py| { let out = lambda_owned.call1(py, args)?; // check if Series, if not, call series constructor on it - SERIES.call1(py, (out,)) + pl_series(py).call1(py, (out,)) }) })? - .to_object(py); + .into_any() + .unbind(); let result = applyer .apply_lambda_with_list_out_type(py, new_lambda, null_count, Some(&series), dt) @@ -116,7 +116,7 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>( py, lambda, null_count, - Some(out.to_object(py).into()), + Some(out.to_owned().unbind().into()), ) .map(|ca| ca.into_series().into()) } @@ -130,14 +130,14 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>( pub trait ApplyLambda<'a> { fn apply_lambda_unknown( &'a self, - _py: Python, + _py: Python<'a>, _lambda: &Bound<'a, PyAny>, ) -> PyResult; // Used to store a struct type fn apply_into_struct( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: AnyValue<'a>, @@ -146,19 +146,19 @@ pub trait ApplyLambda<'a> { /// Apply a lambda with a primitive output type fn apply_lambda_with_primitive_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, ) -> PyResult> where D: PyArrowPrimitiveType, - D::Native: ToPyObject + FromPyObject<'a>; + D::Native: IntoPyObject<'a> + FromPyObject<'a>; /// Apply a lambda with a boolean output type fn apply_lambda_with_bool_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, @@ -167,7 +167,7 @@ pub trait ApplyLambda<'a> { /// Apply a lambda with string output type fn apply_lambda_with_string_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, @@ -176,7 +176,7 @@ pub trait ApplyLambda<'a> { /// Apply a lambda with list output type fn apply_lambda_with_list_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: PyObject, init_null_count: usize, first_value: Option<&Series>, @@ -185,7 +185,7 @@ pub trait ApplyLambda<'a> { fn apply_extract_any_values( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: AnyValue<'a>, @@ -195,7 +195,7 @@ pub trait ApplyLambda<'a> { #[cfg(feature = "object")] fn apply_lambda_with_object_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, @@ -203,34 +203,38 @@ pub trait ApplyLambda<'a> { } pub fn call_lambda<'a, T>( - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, in_val: T, ) -> PyResult> where - T: ToPyObject, + T: IntoPyObject<'a>, { - let arg = PyTuple::new_bound(py, &[in_val]); + let arg = PyTuple::new(py, [in_val])?; lambda.call1(arg) } pub(crate) fn call_lambda_and_extract<'a, 'py, T, S>( - py: Python, + py: Python<'py>, lambda: &'a Bound<'py, PyAny>, in_val: T, ) -> PyResult where - T: ToPyObject, + T: IntoPyObject<'py>, S: FromPyObject<'py>, { call_lambda(py, lambda, in_val).and_then(|out| out.extract::()) } -fn call_lambda_series_out(py: Python, lambda: &Bound, in_val: T) -> PyResult +fn call_lambda_series_out<'py, T>( + py: Python<'py>, + lambda: &Bound, + in_val: T, +) -> PyResult where - T: ToPyObject, + T: IntoPyObject<'py>, { - let arg = PyTuple::new_bound(py, &[in_val]); + let arg = PyTuple::new(py, [in_val])?; let out = lambda.call1(arg)?; let py_series = out.getattr("_s")?; Ok(py_series.extract::().unwrap().series) @@ -241,7 +245,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked { let mut null_count = 0; for opt_v in self.into_iter() { if let Some(v) = opt_v { - let arg = PyTuple::new_bound(py, [v]); + let arg = PyTuple::new(py, [v])?; let out = lambda.call1(arg)?; if out.is_none() { null_count += 1; @@ -296,14 +300,14 @@ impl<'a> ApplyLambda<'a> for BooleanChunked { fn apply_lambda_with_primitive_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, ) -> PyResult> where D: PyArrowPrimitiveType, - D::Native: ToPyObject + FromPyObject<'a>, + D::Native: IntoPyObject<'a> + FromPyObject<'a>, { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { @@ -541,14 +545,18 @@ impl<'a> ApplyLambda<'a> for BooleanChunked { impl<'a, T> ApplyLambda<'a> for ChunkedArray where T: PyArrowPrimitiveType + PolarsNumericType, - T::Native: ToPyObject + FromPyObject<'a>, + T::Native: IntoPyObject<'a> + FromPyObject<'a>, ChunkedArray: IntoSeries, { - fn apply_lambda_unknown(&'a self, py: Python, lambda: &Bound<'a, PyAny>) -> PyResult { + fn apply_lambda_unknown( + &'a self, + py: Python<'a>, + lambda: &Bound<'a, PyAny>, + ) -> PyResult { let mut null_count = 0; for opt_v in self.into_iter() { if let Some(v) = opt_v { - let arg = PyTuple::new_bound(py, [v]); + let arg = PyTuple::new(py, [v])?; let out = lambda.call1(arg)?; if out.is_none() { null_count += 1; @@ -566,7 +574,7 @@ where fn apply_into_struct( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: AnyValue<'a>, @@ -603,14 +611,14 @@ where fn apply_lambda_with_primitive_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, ) -> PyResult> where D: PyArrowPrimitiveType, - D::Native: ToPyObject + FromPyObject<'a>, + D::Native: IntoPyObject<'a> + FromPyObject<'a>, { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { @@ -646,7 +654,7 @@ where fn apply_lambda_with_bool_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, @@ -685,7 +693,7 @@ where fn apply_lambda_with_string_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, @@ -725,7 +733,7 @@ where fn apply_lambda_with_list_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: PyObject, init_null_count: usize, first_value: Option<&Series>, @@ -769,7 +777,7 @@ where fn apply_extract_any_values( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: AnyValue<'a>, @@ -804,7 +812,7 @@ where #[cfg(feature = "object")] fn apply_lambda_with_object_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, @@ -848,7 +856,7 @@ impl<'a> ApplyLambda<'a> for StringChunked { let mut null_count = 0; for opt_v in self.into_iter() { if let Some(v) = opt_v { - let arg = PyTuple::new_bound(py, [v]); + let arg = PyTuple::new(py, [v])?; let out = lambda.call1(arg)?; if out.is_none() { null_count += 1; @@ -903,14 +911,14 @@ impl<'a> ApplyLambda<'a> for StringChunked { fn apply_lambda_with_primitive_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, ) -> PyResult> where D: PyArrowPrimitiveType, - D::Native: ToPyObject + FromPyObject<'a>, + D::Native: IntoPyObject<'a> + FromPyObject<'a>, { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { @@ -1173,7 +1181,7 @@ fn call_series_lambda( impl<'a> ApplyLambda<'a> for ListChunked { fn apply_lambda_unknown(&'a self, py: Python, lambda: &Bound<'a, PyAny>) -> PyResult { - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); let mut null_count = 0; for opt_v in self.into_iter() { if let Some(v) = opt_v { @@ -1210,7 +1218,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { ) -> PyResult { let skip = 1; // get the pypolars module - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); if !self.has_nulls() { let it = self .into_no_null_iter() @@ -1264,17 +1272,17 @@ impl<'a> ApplyLambda<'a> for ListChunked { fn apply_lambda_with_primitive_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, ) -> PyResult> where D: PyArrowPrimitiveType, - D::Native: ToPyObject + FromPyObject<'a>, + D::Native: IntoPyObject<'a> + FromPyObject<'a>, { let skip = usize::from(first_value.is_some()); - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name().clone(), self.len())) } else if !self.has_nulls() { @@ -1334,7 +1342,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { first_value: Option, ) -> PyResult { let skip = usize::from(first_value.is_some()); - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name().clone(), self.len())) } else if !self.has_nulls() { @@ -1395,7 +1403,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { ) -> PyResult { let skip = usize::from(first_value.is_some()); // get the pypolars module - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name().clone(), self.len())) @@ -1457,7 +1465,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { dt: &DataType, ) -> PyResult { let skip = usize::from(first_value.is_some()); - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); let lambda = lambda.bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name().clone(), self.len())) @@ -1465,7 +1473,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { let it = self .into_no_null_iter() .skip(init_null_count + skip) - .map(|val| call_series_lambda(&pypolars, lambda, val)); + .map(|val| call_series_lambda(pypolars, lambda, val)); iterator_to_list( dt, @@ -1479,7 +1487,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { let it = self .into_iter() .skip(init_null_count + skip) - .map(|opt_val| opt_val.and_then(|val| call_series_lambda(&pypolars, lambda, val))); + .map(|opt_val| opt_val.and_then(|val| call_series_lambda(pypolars, lambda, val))); iterator_to_list( dt, it, @@ -1498,7 +1506,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { init_null_count: usize, first_value: AnyValue<'a>, ) -> PyResult { - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); let mut avs = Vec::with_capacity(self.len()); avs.extend(std::iter::repeat(AnyValue::Null).take(init_null_count)); avs.push(first_value); @@ -1545,7 +1553,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { first_value: Option, ) -> PyResult> { let skip = usize::from(first_value.is_some()); - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name().clone(), self.len())) } else if !self.has_nulls() { @@ -1602,7 +1610,7 @@ impl<'a> ApplyLambda<'a> for ListChunked { #[cfg(feature = "dtype-array")] impl<'a> ApplyLambda<'a> for ArrayChunked { fn apply_lambda_unknown(&'a self, py: Python, lambda: &Bound<'a, PyAny>) -> PyResult { - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); let mut null_count = 0; for opt_v in self.into_iter() { if let Some(v) = opt_v { @@ -1639,7 +1647,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { ) -> PyResult { let skip = 1; // get the pypolars module - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); if !self.has_nulls() { let it = self .into_no_null_iter() @@ -1693,17 +1701,17 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { fn apply_lambda_with_primitive_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, ) -> PyResult> where D: PyArrowPrimitiveType, - D::Native: ToPyObject + FromPyObject<'a>, + D::Native: IntoPyObject<'a> + FromPyObject<'a>, { let skip = usize::from(first_value.is_some()); - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name().clone(), self.len())) } else if !self.has_nulls() { @@ -1763,7 +1771,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { first_value: Option, ) -> PyResult { let skip = usize::from(first_value.is_some()); - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name().clone(), self.len())) } else if !self.has_nulls() { @@ -1824,7 +1832,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { ) -> PyResult { let skip = usize::from(first_value.is_some()); // get the pypolars module - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name().clone(), self.len())) @@ -1886,7 +1894,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { dt: &DataType, ) -> PyResult { let skip = usize::from(first_value.is_some()); - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); let lambda = lambda.bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name().clone(), self.len())) @@ -1894,7 +1902,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { let it = self .into_no_null_iter() .skip(init_null_count + skip) - .map(|val| call_series_lambda(&pypolars, lambda, val)); + .map(|val| call_series_lambda(pypolars, lambda, val)); iterator_to_list( dt, @@ -1908,7 +1916,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { let it = self .into_iter() .skip(init_null_count + skip) - .map(|opt_val| opt_val.and_then(|val| call_series_lambda(&pypolars, lambda, val))); + .map(|opt_val| opt_val.and_then(|val| call_series_lambda(pypolars, lambda, val))); iterator_to_list( dt, it, @@ -1927,7 +1935,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { init_null_count: usize, first_value: AnyValue<'a>, ) -> PyResult { - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); let mut avs = Vec::with_capacity(self.len()); avs.extend(std::iter::repeat(AnyValue::Null).take(init_null_count)); avs.push(first_value); @@ -1974,7 +1982,7 @@ impl<'a> ApplyLambda<'a> for ArrayChunked { first_value: Option, ) -> PyResult> { let skip = usize::from(first_value.is_some()); - let pypolars = PyModule::import_bound(py, "polars")?; + let pypolars = polars(py).bind(py); if init_null_count == self.len() { Ok(ChunkedArray::full_null(self.name().clone(), self.len())) } else if !self.has_nulls() { @@ -2034,7 +2042,7 @@ impl<'a> ApplyLambda<'a> for ObjectChunked { let mut null_count = 0; for opt_v in self.into_iter() { if let Some(v) = opt_v { - let arg = PyTuple::new_bound(py, [v]); + let arg = PyTuple::new(py, [v])?; let out = lambda.call1(arg)?; if out.is_none() { null_count += 1; @@ -2052,7 +2060,7 @@ impl<'a> ApplyLambda<'a> for ObjectChunked { fn apply_into_struct( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: AnyValue<'a>, @@ -2077,14 +2085,14 @@ impl<'a> ApplyLambda<'a> for ObjectChunked { fn apply_lambda_with_primitive_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, ) -> PyResult> where D: PyArrowPrimitiveType, - D::Native: ToPyObject + FromPyObject<'a>, + D::Native: IntoPyObject<'a> + FromPyObject<'a>, { let skip = usize::from(first_value.is_some()); if init_null_count == self.len() { @@ -2346,10 +2354,9 @@ impl<'a> ApplyLambda<'a> for StructChunked { first_value: AnyValue<'a>, ) -> PyResult { let skip = 1; - let it = iter_struct(self).skip(init_null_count + skip).map(|val| { - let out = lambda.call1((Wrap(val),)).unwrap(); - Some(out) - }); + let it = iter_struct(self) + .skip(init_null_count + skip) + .map(|val| lambda.call1((Wrap(val),)).ok()); iterator_to_struct( py, it, @@ -2362,14 +2369,14 @@ impl<'a> ApplyLambda<'a> for StructChunked { fn apply_lambda_with_primitive_out_type( &'a self, - py: Python, + py: Python<'a>, lambda: &Bound<'a, PyAny>, init_null_count: usize, first_value: Option, ) -> PyResult> where D: PyArrowPrimitiveType, - D::Native: ToPyObject + FromPyObject<'a>, + D::Native: IntoPyObject<'a> + FromPyObject<'a>, { let skip = usize::from(first_value.is_some()); let it = iter_struct(self) diff --git a/crates/polars-python/src/on_startup.rs b/crates/polars-python/src/on_startup.rs index 8c0b4275b4ba..352f67ae1399 100644 --- a/crates/polars-python/src/on_startup.rs +++ b/crates/polars-python/src/on_startup.rs @@ -6,20 +6,20 @@ use polars_core::chunked_array::object::registry::AnonymousObjectBuilder; use polars_core::chunked_array::object::{registry, set_polars_allow_extension}; use polars_core::error::PolarsError::ComputeError; use polars_error::PolarsWarning; -use pyo3::intern; use pyo3::prelude::*; +use pyo3::{intern, IntoPyObjectExt}; use crate::dataframe::PyDataFrame; use crate::map::lazy::{call_lambda_with_series, ToSeries}; use crate::prelude::ObjectValue; -use crate::py_modules::{POLARS, UTILS}; +use crate::py_modules::{pl_utils, polars}; use crate::Wrap; fn python_function_caller_series(s: Column, lambda: &PyObject) -> PolarsResult { Python::with_gil(|py| { let object = call_lambda_with_series(py, s.clone().take_materialized_series(), lambda) .map_err(|s| ComputeError(format!("{}", s).into()))?; - object.to_series(py, &POLARS, s.name()).map(Column::from) + object.to_series(py, polars(py), s.name()).map(Column::from) }) } @@ -28,7 +28,7 @@ fn python_function_caller_df(df: DataFrame, lambda: &PyObject) -> PolarsResult PolarsResult }); diff --git a/crates/polars-python/src/py_modules.rs b/crates/polars-python/src/py_modules.rs index c193644faef0..006643bef757 100644 --- a/crates/polars-python/src/py_modules.rs +++ b/crates/polars-python/src/py_modules.rs @@ -1,12 +1,18 @@ -use once_cell::sync::Lazy; use pyo3::prelude::*; +use pyo3::sync::GILOnceCell; -pub(crate) static POLARS: Lazy = Lazy::new(|| { - Python::with_gil(|py| PyModule::import_bound(py, "polars").unwrap().to_object(py)) -}); +static POLARS: GILOnceCell> = GILOnceCell::new(); +static UTILS: GILOnceCell = GILOnceCell::new(); +static SERIES: GILOnceCell = GILOnceCell::new(); -pub(crate) static UTILS: Lazy = - Lazy::new(|| Python::with_gil(|py| POLARS.getattr(py, "_utils").unwrap())); +pub(crate) fn polars(py: Python<'_>) -> &Py { + POLARS.get_or_init(py, || py.import("polars").unwrap().unbind()) +} -pub(crate) static SERIES: Lazy = - Lazy::new(|| Python::with_gil(|py| POLARS.getattr(py, "Series").unwrap())); +pub(crate) fn pl_utils(py: Python<'_>) -> &PyObject { + UTILS.get_or_init(py, || polars(py).getattr(py, "_utils").unwrap()) +} + +pub(crate) fn pl_series(py: Python<'_>) -> &PyObject { + SERIES.get_or_init(py, || polars(py).getattr(py, "Series").unwrap()) +} diff --git a/crates/polars-python/src/series/aggregation.rs b/crates/polars-python/src/series/aggregation.rs index c4fe8d3447ec..4f3ea63fff1b 100644 --- a/crates/polars-python/src/series/aggregation.rs +++ b/crates/polars-python/src/series/aggregation.rs @@ -38,141 +38,145 @@ impl PySeries { py.allow_threads(|| self.series.arg_min()) } - fn max(&self, py: Python) -> PyResult { - Ok(Wrap( + fn max<'py>(&self, py: Python<'py>) -> PyResult> { + Wrap( py.allow_threads(|| self.series.max_reduce().map_err(PyPolarsErr::from))? .as_any_value(), ) - .into_py(py)) + .into_pyobject(py) } - fn mean(&self, py: Python) -> PyResult { + fn mean<'py>(&self, py: Python<'py>) -> PyResult> { match self.series.dtype() { - Boolean => Ok(Wrap( + Boolean => Wrap( py.allow_threads(|| self.series.cast(&DataType::UInt8).unwrap().mean_reduce()) .as_any_value(), ) - .into_py(py)), + .into_pyobject(py), // For non-numeric output types we require mean_reduce. - dt if dt.is_temporal() => Ok(Wrap( + dt if dt.is_temporal() => Wrap( py.allow_threads(|| self.series.mean_reduce()) .as_any_value(), ) - .into_py(py)), - _ => Ok(py.allow_threads(|| self.series.mean()).into_py(py)), + .into_pyobject(py), + _ => py + .allow_threads(|| self.series.mean()) + .into_pyobject(py) + .map_err(Into::into), } } - fn median(&self, py: Python) -> PyResult { + fn median<'py>(&self, py: Python<'py>) -> PyResult> { match self.series.dtype() { - Boolean => Ok(Wrap( + Boolean => Wrap( py.allow_threads(|| self.series.cast(&DataType::UInt8).unwrap().median_reduce()) .map_err(PyPolarsErr::from)? .as_any_value(), ) - .into_py(py)), + .into_pyobject(py), // For non-numeric output types we require median_reduce. - dt if dt.is_temporal() => Ok(Wrap( + dt if dt.is_temporal() => Wrap( py.allow_threads(|| self.series.median_reduce()) .map_err(PyPolarsErr::from)? .as_any_value(), ) - .into_py(py)), - _ => Ok(py.allow_threads(|| self.series.median()).into_py(py)), + .into_pyobject(py), + _ => py + .allow_threads(|| self.series.median()) + .into_pyobject(py) + .map_err(Into::into), } } - fn min(&self, py: Python) -> PyResult { - Ok(Wrap( + fn min<'py>(&self, py: Python<'py>) -> PyResult> { + Wrap( py.allow_threads(|| self.series.min_reduce().map_err(PyPolarsErr::from))? .as_any_value(), ) - .into_py(py)) + .into_pyobject(py) } - fn product(&self, py: Python) -> PyResult { - Ok(Wrap( + fn product<'py>(&self, py: Python<'py>) -> PyResult> { + Wrap( py.allow_threads(|| self.series.product().map_err(PyPolarsErr::from))? .as_any_value(), ) - .into_py(py)) + .into_pyobject(py) } - fn quantile( + fn quantile<'py>( &self, - py: Python, + py: Python<'py>, quantile: f64, interpolation: Wrap, - ) -> PyResult { + ) -> PyResult> { let bind = py.allow_threads(|| self.series.quantile_reduce(quantile, interpolation.0)); let sc = bind.map_err(PyPolarsErr::from)?; - Ok(Wrap(sc.as_any_value()).into_py(py)) + Wrap(sc.as_any_value()).into_pyobject(py) } - fn std(&self, py: Python, ddof: u8) -> PyResult { - Ok(Wrap( + fn std<'py>(&self, py: Python<'py>, ddof: u8) -> PyResult> { + Wrap( py.allow_threads(|| self.series.std_reduce(ddof).map_err(PyPolarsErr::from))? .as_any_value(), ) - .into_py(py)) + .into_pyobject(py) } - fn var(&self, py: Python, ddof: u8) -> PyResult { - Ok(Wrap( + fn var<'py>(&self, py: Python<'py>, ddof: u8) -> PyResult> { + Wrap( py.allow_threads(|| self.series.var_reduce(ddof).map_err(PyPolarsErr::from))? .as_any_value(), ) - .into_py(py)) + .into_pyobject(py) } - fn sum(&self, py: Python) -> PyResult { - Ok(Wrap( + fn sum<'py>(&self, py: Python<'py>) -> PyResult> { + Wrap( py.allow_threads(|| self.series.sum_reduce().map_err(PyPolarsErr::from))? .as_any_value(), ) - .into_py(py)) + .into_pyobject(py) } - fn first(&self, py: Python) -> PyObject { - Wrap(py.allow_threads(|| self.series.first()).as_any_value()).into_py(py) + fn first<'py>(&self, py: Python<'py>) -> PyResult> { + Wrap(py.allow_threads(|| self.series.first()).as_any_value()).into_pyobject(py) } - fn last(&self, py: Python) -> PyObject { - Wrap(py.allow_threads(|| self.series.last()).as_any_value()).into_py(py) + fn last<'py>(&self, py: Python<'py>) -> PyResult> { + Wrap(py.allow_threads(|| self.series.last()).as_any_value()).into_pyobject(py) } #[cfg(feature = "approx_unique")] - fn approx_n_unique(&self, py: Python) -> PyResult { - Ok(py - .allow_threads(|| self.series.approx_n_unique().map_err(PyPolarsErr::from))? - .into_py(py)) + fn approx_n_unique(&self, py: Python) -> Result { + py.allow_threads(|| self.series.approx_n_unique().map_err(PyPolarsErr::from)) } #[cfg(feature = "bitwise")] - fn bitwise_and(&self, py: Python) -> PyResult { - Ok(Wrap( + fn bitwise_and<'py>(&self, py: Python<'py>) -> PyResult> { + Wrap( py.allow_threads(|| self.series.and_reduce().map_err(PyPolarsErr::from))? .as_any_value(), ) - .into_py(py)) + .into_pyobject(py) } #[cfg(feature = "bitwise")] - fn bitwise_or(&self, py: Python) -> PyResult { - Ok(Wrap( + fn bitwise_or<'py>(&self, py: Python<'py>) -> PyResult> { + Wrap( py.allow_threads(|| self.series.or_reduce().map_err(PyPolarsErr::from))? .as_any_value(), ) - .into_py(py)) + .into_pyobject(py) } #[cfg(feature = "bitwise")] - fn bitwise_xor(&self, py: Python) -> PyResult { - Ok(Wrap( + fn bitwise_xor<'py>(&self, py: Python<'py>) -> PyResult> { + Wrap( py.allow_threads(|| self.series.xor_reduce().map_err(PyPolarsErr::from))? .as_any_value(), ) - .into_py(py)) + .into_pyobject(py) } } diff --git a/crates/polars-python/src/series/buffers.rs b/crates/polars-python/src/series/buffers.rs index 2510476cfbed..960b44949337 100644 --- a/crates/polars-python/src/series/buffers.rs +++ b/crates/polars-python/src/series/buffers.rs @@ -22,6 +22,7 @@ use polars::prelude::*; use polars_core::{with_match_physical_numeric_polars_type, with_match_physical_numeric_type}; use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; +use pyo3::types::PyTuple; use super::{PySeries, ToSeries}; use crate::conversion::Wrap; @@ -33,9 +34,13 @@ struct BufferInfo { offset: usize, length: usize, } -impl IntoPy for BufferInfo { - fn into_py(self, py: Python<'_>) -> PyObject { - (self.pointer, self.offset, self.length).to_object(py) +impl<'py> IntoPyObject<'py> for BufferInfo { + type Target = PyTuple; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + (self.pointer, self.offset, self.length).into_pyobject(py) } } impl<'a> FromPyObject<'a> for BufferInfo { @@ -173,7 +178,6 @@ impl PySeries { /// Construct a PySeries from information about its underlying buffer. #[staticmethod] unsafe fn _from_buffer( - py: Python, dtype: Wrap, buffer_info: BufferInfo, owner: &Bound<'_, PyAny>, @@ -184,7 +188,7 @@ impl PySeries { offset, length, } = buffer_info; - let owner = owner.to_object(py); + let owner = owner.to_owned().unbind(); let arr_boxed = match dtype { dt if dt.is_numeric() => { diff --git a/crates/polars-python/src/series/construction.rs b/crates/polars-python/src/series/construction.rs index 70977696ca3c..a15773a21e3f 100644 --- a/crates/polars-python/src/series/construction.rs +++ b/crates/polars-python/src/series/construction.rs @@ -23,13 +23,8 @@ macro_rules! init_method { #[pymethods] impl PySeries { #[staticmethod] - fn $name( - py: Python, - name: &str, - array: &Bound>, - _strict: bool, - ) -> Self { - mmap_numpy_array(py, name, array) + fn $name(name: &str, array: &Bound>, _strict: bool) -> Self { + mmap_numpy_array(name, array) } } }; @@ -44,14 +39,10 @@ init_method!(new_u16, u16); init_method!(new_u32, u32); init_method!(new_u64, u64); -fn mmap_numpy_array( - py: Python, - name: &str, - array: &Bound>, -) -> PySeries { +fn mmap_numpy_array(name: &str, array: &Bound>) -> PySeries { let vals = unsafe { array.as_slice().unwrap() }; - let arr = unsafe { arrow::ffi::mmap::slice_and_owner(vals, array.to_object(py)) }; + let arr = unsafe { arrow::ffi::mmap::slice_and_owner(vals, array.clone().unbind()) }; Series::from_arrow(name.into(), arr.to_boxed()) .unwrap() .into() @@ -78,7 +69,7 @@ impl PySeries { }); ca.with_name(name.into()).into_series().into() } else { - mmap_numpy_array(py, name, array) + mmap_numpy_array(name, array) } } @@ -94,7 +85,7 @@ impl PySeries { }); ca.with_name(name.into()).into_series().into() } else { - mmap_numpy_array(py, name, array) + mmap_numpy_array(name, array) } } } @@ -106,7 +97,7 @@ impl PySeries { let len = values.len()?; let mut builder = BooleanChunkedBuilder::new(name.into(), len); - for res in values.iter()? { + for res in values.try_iter()? { let value = res?; if value.is_none() { builder.append_null() @@ -131,7 +122,7 @@ where let len = values.len()?; let mut builder = PrimitiveChunkedBuilder::::new(name.into(), len); - for res in values.iter()? { + for res in values.try_iter()? { let value = res?; if value.is_none() { builder.append_null() @@ -177,7 +168,7 @@ fn convert_to_avs<'a>( allow_object: bool, ) -> PyResult>> { values - .iter()? + .try_iter()? .map(|v| py_object_to_any_value(&(v?).as_borrowed(), strict, allow_object)) .collect() } @@ -187,7 +178,7 @@ impl PySeries { #[staticmethod] fn new_from_any_values(name: &str, values: &Bound, strict: bool) -> PyResult { let any_values_result = values - .iter()? + .try_iter()? .map(|v| py_object_to_any_value(&(v?).as_borrowed(), strict, true)) .collect::>>(); let result = any_values_result.and_then(|avs| { @@ -203,13 +194,8 @@ impl PySeries { if !strict && result.is_err() { return Python::with_gil(|py| { let objects = values - .iter()? - .map(|v| { - let obj = ObjectValue { - inner: v?.to_object(py), - }; - Ok(obj) - }) + .try_iter()? + .map(|v| v?.extract()) .collect::>>()?; Ok(Self::new_object(py, name, objects, strict)) }); @@ -240,7 +226,7 @@ impl PySeries { let len = values.len()?; let mut builder = StringChunkedBuilder::new(name.into(), len); - for res in values.iter()? { + for res in values.try_iter()? { let value = res?; if value.is_none() { builder.append_null() @@ -260,7 +246,7 @@ impl PySeries { let len = values.len()?; let mut builder = BinaryChunkedBuilder::new(name.into(), len); - for res in values.iter()? { + for res in values.try_iter()? { let value = res?; if value.is_none() { builder.append_null() diff --git a/crates/polars-python/src/series/export.rs b/crates/polars-python/src/series/export.rs index 79bf3ed549c3..f9c76e226ee3 100644 --- a/crates/polars-python/src/series/export.rs +++ b/crates/polars-python/src/series/export.rs @@ -1,8 +1,10 @@ use polars_core::prelude::*; use pyo3::prelude::*; use pyo3::types::{PyCapsule, PyList}; +use pyo3::IntoPyObjectExt; use super::PySeries; +use crate::error::PyPolarsErr; use crate::interop; use crate::interop::arrow::to_py::series_to_stream; use crate::prelude::*; @@ -11,148 +13,143 @@ use crate::prelude::*; impl PySeries { /// Convert this Series to a Python list. /// This operation copies data. - pub fn to_list(&self) -> PyObject { - Python::with_gil(|py| { - let series = &self.series; + pub fn to_list<'py>(&self, py: Python<'py>) -> PyResult> { + let series = &self.series; - fn to_list_recursive(py: Python, series: &Series) -> PyObject { - let pylist = match series.dtype() { - DataType::Boolean => PyList::new_bound(py, series.bool().unwrap()), - DataType::UInt8 => PyList::new_bound(py, series.u8().unwrap()), - DataType::UInt16 => PyList::new_bound(py, series.u16().unwrap()), - DataType::UInt32 => PyList::new_bound(py, series.u32().unwrap()), - DataType::UInt64 => PyList::new_bound(py, series.u64().unwrap()), - DataType::Int8 => PyList::new_bound(py, series.i8().unwrap()), - DataType::Int16 => PyList::new_bound(py, series.i16().unwrap()), - DataType::Int32 => PyList::new_bound(py, series.i32().unwrap()), - DataType::Int64 => PyList::new_bound(py, series.i64().unwrap()), - DataType::Int128 => PyList::new_bound(py, series.i128().unwrap()), - DataType::Float32 => PyList::new_bound(py, series.f32().unwrap()), - DataType::Float64 => PyList::new_bound(py, series.f64().unwrap()), - DataType::Categorical(_, _) | DataType::Enum(_, _) => { - PyList::new_bound(py, series.categorical().unwrap().iter_str()) - }, - #[cfg(feature = "object")] - DataType::Object(_, _) => { - let v = PyList::empty_bound(py); - for i in 0..series.len() { - let obj: Option<&ObjectValue> = - series.get_object(i).map(|any| any.into()); - let val = obj.to_object(py); - - v.append(val).unwrap(); - } - v - }, - DataType::List(_) => { - let v = PyList::empty_bound(py); - let ca = series.list().unwrap(); - for opt_s in ca.amortized_iter() { - match opt_s { - None => { - v.append(py.None()).unwrap(); - }, - Some(s) => { - let pylst = to_list_recursive(py, s.as_ref()); - v.append(pylst).unwrap(); - }, - } + fn to_list_recursive<'py>(py: Python<'py>, series: &Series) -> PyResult> { + let pylist = match series.dtype() { + DataType::Boolean => PyList::new(py, series.bool().map_err(PyPolarsErr::from)?)?, + DataType::UInt8 => PyList::new(py, series.u8().map_err(PyPolarsErr::from)?)?, + DataType::UInt16 => PyList::new(py, series.u16().map_err(PyPolarsErr::from)?)?, + DataType::UInt32 => PyList::new(py, series.u32().map_err(PyPolarsErr::from)?)?, + DataType::UInt64 => PyList::new(py, series.u64().map_err(PyPolarsErr::from)?)?, + DataType::Int8 => PyList::new(py, series.i8().map_err(PyPolarsErr::from)?)?, + DataType::Int16 => PyList::new(py, series.i16().map_err(PyPolarsErr::from)?)?, + DataType::Int32 => PyList::new(py, series.i32().map_err(PyPolarsErr::from)?)?, + DataType::Int64 => PyList::new(py, series.i64().map_err(PyPolarsErr::from)?)?, + DataType::Int128 => PyList::new(py, series.i128().map_err(PyPolarsErr::from)?)?, + DataType::Float32 => PyList::new(py, series.f32().map_err(PyPolarsErr::from)?)?, + DataType::Float64 => PyList::new(py, series.f64().map_err(PyPolarsErr::from)?)?, + DataType::Categorical(_, _) | DataType::Enum(_, _) => PyList::new( + py, + series.categorical().map_err(PyPolarsErr::from)?.iter_str(), + )?, + #[cfg(feature = "object")] + DataType::Object(_, _) => { + let v = PyList::empty(py); + for i in 0..series.len() { + let obj: Option<&ObjectValue> = series.get_object(i).map(|any| any.into()); + v.append(obj)?; + } + v + }, + DataType::List(_) => { + let v = PyList::empty(py); + let ca = series.list().map_err(PyPolarsErr::from)?; + for opt_s in ca.amortized_iter() { + match opt_s { + None => { + v.append(py.None())?; + }, + Some(s) => { + let pylst = to_list_recursive(py, s.as_ref())?; + v.append(pylst)?; + }, } - v - }, - DataType::Array(_, _) => { - let v = PyList::empty_bound(py); - let ca = series.array().unwrap(); - for opt_s in ca.amortized_iter() { - match opt_s { - None => { - v.append(py.None()).unwrap(); - }, - Some(s) => { - let pylst = to_list_recursive(py, s.as_ref()); - v.append(pylst).unwrap(); - }, - } + } + v + }, + DataType::Array(_, _) => { + let v = PyList::empty(py); + let ca = series.array().map_err(PyPolarsErr::from)?; + for opt_s in ca.amortized_iter() { + match opt_s { + None => { + v.append(py.None())?; + }, + Some(s) => { + let pylst = to_list_recursive(py, s.as_ref())?; + v.append(pylst)?; + }, } - v - }, - DataType::Date => { - let ca = series.date().unwrap(); - return Wrap(ca).to_object(py); - }, - DataType::Time => { - let ca = series.time().unwrap(); - return Wrap(ca).to_object(py); - }, - DataType::Datetime(_, _) => { - let ca = series.datetime().unwrap(); - return Wrap(ca).to_object(py); - }, - DataType::Decimal(_, _) => { - let ca = series.decimal().unwrap(); - return Wrap(ca).to_object(py); - }, - DataType::String => { - let ca = series.str().unwrap(); - return Wrap(ca).to_object(py); - }, - DataType::Struct(_) => { - let ca = series.struct_().unwrap(); - return Wrap(ca).to_object(py); - }, - DataType::Duration(_) => { - let ca = series.duration().unwrap(); - return Wrap(ca).to_object(py); - }, - DataType::Binary => { - let ca = series.binary().unwrap(); - return Wrap(ca).to_object(py); - }, - DataType::Null => { - let null: Option = None; - let n = series.len(); - let iter = std::iter::repeat(null).take(n); - use std::iter::{Repeat, Take}; - struct NullIter { - iter: Take>>, - n: usize, - } - impl Iterator for NullIter { - type Item = Option; + } + v + }, + DataType::Date => { + let ca = series.date().map_err(PyPolarsErr::from)?; + return Wrap(ca).into_bound_py_any(py); + }, + DataType::Time => { + let ca = series.time().map_err(PyPolarsErr::from)?; + return Wrap(ca).into_bound_py_any(py); + }, + DataType::Datetime(_, _) => { + let ca = series.datetime().map_err(PyPolarsErr::from)?; + return Wrap(ca).into_bound_py_any(py); + }, + DataType::Decimal(_, _) => { + let ca = series.decimal().map_err(PyPolarsErr::from)?; + return Wrap(ca).into_bound_py_any(py); + }, + DataType::String => { + let ca = series.str().map_err(PyPolarsErr::from)?; + return Wrap(ca).into_bound_py_any(py); + }, + DataType::Struct(_) => { + let ca = series.struct_().map_err(PyPolarsErr::from)?; + return Wrap(ca).into_bound_py_any(py); + }, + DataType::Duration(_) => { + let ca = series.duration().map_err(PyPolarsErr::from)?; + return Wrap(ca).into_bound_py_any(py); + }, + DataType::Binary => { + let ca = series.binary().map_err(PyPolarsErr::from)?; + return Wrap(ca).into_bound_py_any(py); + }, + DataType::Null => { + let null: Option = None; + let n = series.len(); + let iter = std::iter::repeat(null).take(n); + use std::iter::{Repeat, Take}; + struct NullIter { + iter: Take>>, + n: usize, + } + impl Iterator for NullIter { + type Item = Option; - fn next(&mut self) -> Option { - self.iter.next() - } - fn size_hint(&self) -> (usize, Option) { - (self.n, Some(self.n)) - } + fn next(&mut self) -> Option { + self.iter.next() + } + fn size_hint(&self) -> (usize, Option) { + (self.n, Some(self.n)) } - impl ExactSizeIterator for NullIter {} + } + impl ExactSizeIterator for NullIter {} - PyList::new_bound(py, NullIter { iter, n }) - }, - DataType::Unknown(_) => { - panic!("to_list not implemented for unknown") - }, - DataType::BinaryOffset => { - unreachable!() - }, - }; - pylist.to_object(py) - } + PyList::new(py, NullIter { iter, n })? + }, + DataType::Unknown(_) => { + panic!("to_list not implemented for unknown") + }, + DataType::BinaryOffset => { + unreachable!() + }, + }; + Ok(pylist.into_any()) + } - let pylist = to_list_recursive(py, series); - pylist.to_object(py) - }) + to_list_recursive(py, series) } /// Return the underlying Arrow array. #[allow(clippy::wrong_self_convention)] fn to_arrow(&mut self, py: Python, compat_level: PyCompatLevel) -> PyResult { self.rechunk(py, true); - let pyarrow = py.import_bound("pyarrow")?; + let pyarrow = py.import("pyarrow")?; - interop::arrow::to_py::to_py_array(self.series.to_arrow(0, compat_level.0), py, &pyarrow) + interop::arrow::to_py::to_py_array(self.series.to_arrow(0, compat_level.0), &pyarrow) } #[allow(unused_variables)] diff --git a/crates/polars-python/src/series/general.rs b/crates/polars-python/src/series/general.rs index 431343519d47..25a9473b3f0f 100644 --- a/crates/polars-python/src/series/general.rs +++ b/crates/polars-python/src/series/general.rs @@ -7,14 +7,14 @@ use polars_row::RowEncodingOptions; use pyo3::exceptions::{PyIndexError, PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyBytes; -use pyo3::Python; +use pyo3::{IntoPyObjectExt, Python}; use self::row_encode::get_row_encoding_dictionary; use super::PySeries; use crate::dataframe::PyDataFrame; use crate::error::PyPolarsErr; use crate::prelude::*; -use crate::py_modules::POLARS; +use crate::py_modules::polars; #[pymethods] impl PySeries { @@ -68,15 +68,13 @@ impl PySeries { } #[cfg(feature = "object")] - fn get_object(&self, index: usize) -> PyObject { - Python::with_gil(|py| { - if matches!(self.series.dtype(), DataType::Object(_, _)) { - let obj: Option<&ObjectValue> = self.series.get_object(index).map(|any| any.into()); - obj.to_object(py) - } else { - py.None() - } - }) + fn get_object<'py>(&self, py: Python<'py>, index: usize) -> PyResult> { + if matches!(self.series.dtype(), DataType::Object(_, _)) { + let obj: Option<&ObjectValue> = self.series.get_object(index).map(|any| any.into()); + Ok(obj.into_pyobject(py)?) + } else { + Ok(py.None().into_bound(py)) + } } #[cfg(feature = "dtype-array")] @@ -135,20 +133,13 @@ impl PySeries { Err(e) => return Err(PyPolarsErr::from(e).into()), }; - let out = match av { + match av { AnyValue::List(s) | AnyValue::Array(s, _) => { let pyseries = PySeries::new(s); - let out = POLARS - .getattr(py, "wrap_s") - .unwrap() - .call1(py, (pyseries,)) - .unwrap(); - out.into_py(py) + polars(py).getattr(py, "wrap_s")?.call1(py, (pyseries,)) }, - _ => Wrap(av).into_py(py), - }; - - Ok(out) + _ => Wrap(av).into_py_any(py), + } } /// Get a value by index, allowing negative indices. @@ -200,8 +191,8 @@ impl PySeries { self.series.rename(name.into()); } - fn dtype(&self, py: Python) -> PyObject { - Wrap(self.series.dtype().clone()).to_object(py) + fn dtype<'py>(&self, py: Python<'py>) -> PyResult> { + Wrap(self.series.dtype().clone()).into_pyobject(py) } fn set_sorted_flag(&self, descending: bool) -> Self { @@ -374,7 +365,7 @@ impl PySeries { py.allow_threads(|| self.series.shrink_to_fit()); } - fn dot(&self, other: &PySeries, py: Python) -> PyResult { + fn dot<'py>(&self, other: &PySeries, py: Python<'py>) -> PyResult> { let lhs_dtype = self.series.dtype(); let rhs_dtype = other.series.dtype(); @@ -395,11 +386,11 @@ impl PySeries { .into() }; - Ok(Wrap(result).into_py(py)) + Wrap(result).into_pyobject(py) } #[cfg(feature = "ipc_streaming")] - fn __getstate__(&self, py: Python) -> PyResult { + fn __getstate__<'py>(&self, py: Python<'py>) -> PyResult> { // Used in pickle/pickling let mut buf: Vec = vec![]; // IPC only support DataFrames so we need to convert it @@ -408,7 +399,7 @@ impl PySeries { .with_compat_level(CompatLevel::newest()) .finish(&mut df) .expect("ipc writer"); - Ok(PyBytes::new_bound(py, &buf).to_object(py)) + Ok(PyBytes::new(py, &buf)) } #[cfg(feature = "ipc_streaming")] @@ -474,7 +465,7 @@ impl PySeries { fn get_chunks(&self) -> PyResult> { Python::with_gil(|py| { - let wrap_s = py_modules::POLARS.getattr(py, "wrap_s").unwrap(); + let wrap_s = py_modules::polars(py).getattr(py, "wrap_s").unwrap(); flatten_series(&self.series) .into_iter() .map(|s| wrap_s.call1(py, (Self::new(s),))) diff --git a/crates/polars-python/src/series/map.rs b/crates/polars-python/src/series/map.rs index f003096f7dc9..be2a4e664fa7 100644 --- a/crates/polars-python/src/series/map.rs +++ b/crates/polars-python/src/series/map.rs @@ -6,7 +6,7 @@ use super::PySeries; use crate::error::PyPolarsErr; use crate::map::series::{call_lambda_and_extract, ApplyLambda}; use crate::prelude::*; -use crate::py_modules::SERIES; +use crate::py_modules::pl_series; use crate::{apply_method_all_arrow_series2, raise_err}; #[pymethods] @@ -226,16 +226,17 @@ impl PySeries { }, Some(DataType::List(inner)) => { // Make sure the function returns a Series of the correct data type. - let function_owned = function.to_object(py); - let dtype_py = Wrap((*inner).clone()).to_object(py); + let function_owned = function.clone().unbind(); + let dtype_py = Wrap((*inner).clone()); let function_wrapped = - PyCFunction::new_closure_bound(py, None, None, move |args, _kwargs| { + PyCFunction::new_closure(py, None, None, move |args, _kwargs| { Python::with_gil(|py| { let out = function_owned.call1(py, args)?; - SERIES.call1(py, ("", out, dtype_py.clone_ref(py))) + pl_series(py).call1(py, ("", out, &dtype_py)) }) })? - .to_object(py); + .into_any() + .unbind(); let ca = dispatch_apply!( series, diff --git a/crates/polars-python/src/series/numpy_ufunc.rs b/crates/polars-python/src/series/numpy_ufunc.rs index 5df438686447..471e0e19c89e 100644 --- a/crates/polars-python/src/series/numpy_ufunc.rs +++ b/crates/polars-python/src/series/numpy_ufunc.rs @@ -35,7 +35,7 @@ unsafe fn aligned_array( let ptr = PY_ARRAY_API.PyArray_NewFromDescr( py, PY_ARRAY_API.get_type_object(py, npyffi::NpyTypes::PyArray_Type), - T::get_dtype_bound(py).into_dtype_ptr(), + T::get_dtype(py).into_dtype_ptr(), dims.ndim_cint(), dims.as_dims_ptr(), strides.as_ptr() as *mut _, // strides @@ -83,9 +83,8 @@ macro_rules! impl_ufuncs { if !allocate_out { // We're not going to allocate the output array. // Instead, we'll let the ufunc do it. - let result = lambda.call1((PyNone::get_bound(py),))?; - let series_factory = - PyModule::import_bound(py, "polars")?.getattr("Series")?; + let result = lambda.call1((PyNone::get(py),))?; + let series_factory = crate::py_modules::pl_series(py).bind(py); return series_factory .call((self.name(), result), None)? .getattr("_s")? @@ -98,7 +97,7 @@ macro_rules! impl_ufuncs { debug_assert_eq!(get_refcnt(&out_array), 1); // inserting it in a tuple increase the reference count by 1. - let args = PyTuple::new_bound(py, &[out_array.clone()]); + let args = PyTuple::new(py, &[out_array.clone()])?; debug_assert_eq!(get_refcnt(&out_array), 2); // whatever the result, we must take the leaked memory ownership back diff --git a/crates/polars-python/src/sql.rs b/crates/polars-python/src/sql.rs index 65312688fc66..8807bf43dcaa 100644 --- a/crates/polars-python/src/sql.rs +++ b/crates/polars-python/src/sql.rs @@ -4,7 +4,7 @@ use pyo3::prelude::*; use crate::error::PyPolarsErr; use crate::PyLazyFrame; -#[pyclass] +#[pyclass(unsendable)] #[repr(transparent)] #[derive(Clone)] pub struct PySQLContext { diff --git a/crates/polars-utils/src/python_function.rs b/crates/polars-utils/src/python_function.rs index 621eace590f7..a3f20e8717db 100644 --- a/crates/polars-utils/src/python_function.rs +++ b/crates/polars-utils/src/python_function.rs @@ -78,7 +78,7 @@ impl TrySerializeToBytes for PythonFunction { pub fn serialize_pyobject_with_cloudpickle_fallback(py_object: &PyObject) -> PolarsResult> { Python::with_gil(|py| { - let pickle = PyModule::import_bound(py, "pickle") + let pickle = PyModule::import(py, "pickle") .expect("unable to import 'pickle'") .getattr("dumps") .unwrap(); @@ -88,7 +88,7 @@ pub fn serialize_pyobject_with_cloudpickle_fallback(py_object: &PyObject) -> Pol let (dumped, used_cloudpickle) = if let Ok(v) = dumped { (v, false) } else { - let cloudpickle = PyModule::import_bound(py, "cloudpickle") + let cloudpickle = PyModule::import(py, "cloudpickle") .map_err(from_pyerr)? .getattr("dumps") .unwrap(); @@ -115,11 +115,11 @@ pub fn deserialize_pyobject_bytes_maybe_cloudpickle From>( let bytes = rem; Python::with_gil(|py| { - let pickle = PyModule::import_bound(py, "pickle") + let pickle = PyModule::import(py, "pickle") .expect("unable to import 'pickle'") .getattr("loads") .unwrap(); - let arg = (PyBytes::new_bound(py, bytes),); + let arg = (PyBytes::new(py, bytes),); let pyany_bound = pickle.call1(arg).map_err(from_pyerr)?; Ok(PyObject::from(pyany_bound).into()) }) @@ -213,7 +213,7 @@ mod serde_wrap { /// Get the [minor, micro] Python3 version from the `sys` module. fn get_python3_version() -> [u8; 2] { Python::with_gil(|py| { - let version_info = PyModule::import_bound(py, "sys") + let version_info = PyModule::import(py, "sys") .unwrap() .getattr("version_info") .unwrap(); diff --git a/py-polars/src/lib.rs b/py-polars/src/lib.rs index a62da6cf3366..58adee79e260 100644 --- a/py-polars/src/lib.rs +++ b/py-polars/src/lib.rs @@ -292,100 +292,85 @@ fn polars(py: Python, m: &Bound) -> PyResult<()> { .unwrap(); // Exceptions - Errors - m.add( - "PolarsError", - py.get_type_bound::(), - ) - .unwrap(); + m.add("PolarsError", py.get_type::()) + .unwrap(); m.add( "ColumnNotFoundError", - py.get_type_bound::(), - ) - .unwrap(); - m.add( - "ComputeError", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); + m.add("ComputeError", py.get_type::()) + .unwrap(); m.add( "DuplicateError", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); m.add( "InvalidOperationError", - py.get_type_bound::(), - ) - .unwrap(); - m.add( - "NoDataError", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); + m.add("NoDataError", py.get_type::()) + .unwrap(); m.add( "OutOfBoundsError", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); m.add( "SQLInterfaceError", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); m.add( "SQLSyntaxError", - py.get_type_bound::(), - ) - .unwrap(); - m.add( - "SchemaError", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); + m.add("SchemaError", py.get_type::()) + .unwrap(); m.add( "SchemaFieldNotFoundError", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); - m.add("ShapeError", py.get_type_bound::()) + m.add("ShapeError", py.get_type::()) .unwrap(); m.add( "StringCacheMismatchError", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); m.add( "StructFieldNotFoundError", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); // Exceptions - Warnings - m.add( - "PolarsWarning", - py.get_type_bound::(), - ) - .unwrap(); + m.add("PolarsWarning", py.get_type::()) + .unwrap(); m.add( "PerformanceWarning", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); m.add( "CategoricalRemappingWarning", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); m.add( "MapWithoutReturnDtypeWarning", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); // Exceptions - Panic m.add( "PanicException", - py.get_type_bound::(), + py.get_type::(), ) .unwrap(); diff --git a/py-polars/tests/unit/datatypes/test_duration.py b/py-polars/tests/unit/datatypes/test_duration.py index e690e5e92dd5..9d2008bff1bf 100644 --- a/py-polars/tests/unit/datatypes/test_duration.py +++ b/py-polars/tests/unit/datatypes/test_duration.py @@ -3,7 +3,6 @@ import pytest import polars as pl -from polars.exceptions import PanicException from polars.testing import assert_frame_equal @@ -168,7 +167,7 @@ def test_series_duration_std_var() -> None: def test_series_duration_var_overflow() -> None: s = pl.Series([timedelta(days=10), timedelta(days=20), timedelta(days=40)]) - with pytest.raises(PanicException, match="OverflowError"): + with pytest.raises(OverflowError): s.var()