pyo3-polars 0.7.0 <-> matches polars 0.33.2

Cargo.toml

@@ -1,15 +1,15 @@
 [workspace]
 resolver = "2"
 members = [
-    "example/derive_expression/expression_lib",
-    "example/extend_polars_python_dispatch/extend_polars",
-    "pyo3-polars",
-    "pyo3-polars-derive",
+  "example/derive_expression/expression_lib",
+  "example/extend_polars_python_dispatch/extend_polars",
+  "pyo3-polars",
+  "pyo3-polars-derive",
 ]
 
 [workspace.dependencies]
-polars = {version = "0.33.2", default-features=false}
-polars-core = {version = "0.33.2", default-features=false}
-polars-ffi = {ersion = "0.33.2", default-features=false}
-polars-plan = {version = "0.33.2", default-feautres=false}
-polars-lazy = {version = "0.33.2", default-features=false}
+polars = { version = "0.33.2", default-features = false }
+polars-core = { version = "0.33.2", default-features = false }
+polars-ffi = { version = "0.33.2", default-features = false }
+polars-plan = { version = "0.33.2", default-feautres = false }
+polars-lazy = { version = "0.33.2", default-features = false }

README.md

@@ -1,11 +1,80 @@
 ## 1. Shared library plugins for Polars
+
 This is new functionality and not entirely stable, but should be preferred over `2.` as this
 will circumvent the GIL and will be the way we want to support extending polars.
 
-See more in `examples/derive_expression`.
+Parallelism and optimizations are managed by the default polars runtime. That runtime will call into the plugin function.
+The plugin functions are compiled separately.
+
+We can therefore keep polars more lean and maybe add support for a `polars-distance`, `polars-geo`, `polars-ml`, etc. Those can then have specialized expressions and don't have to worry as much for code bloat as they can be optionally installed.
+
+The idea is that you define an expression in another Rust crate with a proc_macro `polars_expr`.
+
+That macro can have the following attributes:
+
+- `output_type` -> to define the output type of that expression
+- `type_func` -> to define a function that computes the output type based on input types.
+
+Here is an example of a `String` conversion expression that converts any string to [pig latin](https://en.wikipedia.org/wiki/Pig_Latin):
+
+```rust
+fn pig_latin_str(value: &str, output: &mut String) {
+    if let Some(first_char) = value.chars().next() {
+        write!(output, "{}{}ay", &value[1..], first_char).unwrap()
+    }
+}
+
+#[polars_expr(output_type=Utf8)]
+fn pig_latinnify(inputs: &[Series]) -> PolarsResult<Series> {
+    let ca = inputs[0].utf8()?;
+    let out: Utf8Chunked = ca.apply_to_buffer(pig_latin_str);
+    Ok(out.into_series())
+}
+```
+
+On the python side this expression can then be registered under a namespace:
+
+```python
+import polars as pl
+from polars.utils.udfs import _get_shared_lib_location
+
+lib = _get_shared_lib_location(__file__)
+
+
+@pl.api.register_expr_namespace("language")
+class Language:
+    def __init__(self, expr: pl.Expr):
+        self._expr = expr
+
+    def pig_latinnify(self) -> pl.Expr:
+        return self._expr._register_plugin(
+            lib=lib,
+            symbol="pig_latinnify",
+            is_elementwise=True,
+        )
+```
+
+Compile/ship and then it is ready to use:
+
+```python
+import polars as pl
+from expression_lib import Language
+
+df = pl.DataFrame({
+    "names": ["Richard", "Alice", "Bob"],
+})
+
+
+out = df.with_columns(
+   pig_latin = pl.col("names").language.pig_latinnify()
+)
+```
+
+See the full example in [example/derive_expression]: https://github.com/pola-rs/pyo3-polars/tree/plugin/example/derive_expression
 
 ## 2. Pyo3 extensions for Polars
-  <a href="https://crates.io/crates/pyo3-polars">
+
+<a href="https://crates.io/crates/pyo3-polars">
     <img src="https://img.shields.io/crates/v/pyo3-polars.svg"/>
   </a>
 
@@ -41,9 +110,10 @@ shape: (2, 1)
 ```
 
 ## Compile for release
+
 `$ make install-release`
 
 # What to expect
 
 This crate offers a `PySeries` and a `PyDataFrame` which are simple wrapper around `Series` and `DataFrame`. The
-advantage of these wrappers is that they can be converted to and from python as they implement `FromPyObject` and `IntoPy`.
+advantage of these wrappers is that they can be converted to and from python as they implement `FromPyObject` and `IntoPy`.

dprint.json

@@ -0,0 +1,17 @@
+{
+  "json": {
+  },
+  "markdown": {
+  },
+  "toml": {
+  },
+  "includes": ["**/*.{json,md,toml}"],
+  "excludes": [
+    "**/*-lock.json"
+  ],
+  "plugins": [
+    "https://plugins.dprint.dev/json-0.17.4.wasm",
+    "https://plugins.dprint.dev/markdown-0.16.1.wasm",
+    "https://plugins.dprint.dev/toml-0.5.4.wasm"
+  ]
+}

example/derive_expression/expression_lib/Cargo.toml

@@ -9,7 +9,7 @@ name = "expression_lib"
 crate-type = ["cdylib"]
 
 [dependencies]
+polars = { workspace = true, features = ["fmt"], default-features = false }
+polars-plan = { workspace = true, default-features = false }
 pyo3 = { version = "0.19.0", features = ["extension-module"] }
-pyo3-polars = { version = "*", path = "../../../pyo3-polars", features=["derive"] }
-polars = { workspace = true, features = ["fmt"], default-features=false }
-polars-plan = { workspace = true, default-features=false }
+pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive"] }

example/derive_expression/expression_lib/pyproject.toml

@@ -6,9 +6,7 @@ build-backend = "maturin"
 name = "expression_lib"
 requires-python = ">=3.8"
 classifiers = [
-    "Programming Language :: Rust",
-    "Programming Language :: Python :: Implementation :: CPython",
-    "Programming Language :: Python :: Implementation :: PyPy",
+  "Programming Language :: Rust",
+  "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: Implementation :: PyPy",
 ]
-
-

example/extend_polars_python_dispatch/extend_polars/Cargo.toml

@@ -9,9 +9,9 @@ name = "extend_polars"
 crate-type = ["cdylib"]
 
 [dependencies]
+polars = { workspace = true, features = ["fmt"] }
+polars-core = { workspace = true }
+polars-lazy = { workspace = true }
 pyo3 = { version = "0.19", features = ["extension-module"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["lazy"] }
-polars-core = { workspace = true }
-polars-lazy =  {workspace = true}
-polars = { workspace = true, features = ["fmt"] }
 rayon = "1.6"

example/extend_polars_python_dispatch/extend_polars/pyproject.toml

@@ -6,9 +6,7 @@ build-backend = "maturin"
 name = "extend_polars"
 requires-python = ">=3.7"
 classifiers = [
-    "Programming Language :: Rust",
-    "Programming Language :: Python :: Implementation :: CPython",
-    "Programming Language :: Python :: Implementation :: PyPy",
+  "Programming Language :: Rust",
+  "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: Implementation :: PyPy",
 ]
-
-

pyo3-polars-derive/Cargo.toml

@@ -2,6 +2,10 @@
 name = "pyo3-polars-derive"
 version = "0.1.0"
 edition = "2021"
+license = "MIT"
+readme = "README.md"
+repository = "https://github.com/pola-rs/pyo3-polars"
+description = "Proc-macro's for pyo3-polars"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -13,12 +17,12 @@ name = "tests"
 path = "tests/run.rs"
 
 [dependencies]
-polars-core = {workspace = true}
-polars-ffi = {workspace = true}
-polars-plan = {workspace = true}
-syn = {version = "2", features= ["full", "extra-traits"]}
-quote = "1.0"
+polars-core = { workspace = true }
+polars-ffi = { workspace = true }
+polars-plan = { workspace = true }
 proc-macro2 = "1.0"
+quote = "1.0"
+syn = { version = "2", features = ["full", "extra-traits"] }
 
 [dev-dependencies]
 trybuild = { version = "1", features = ["diff"] }

pyo3-polars-derive/README.md

@@ -0,0 +1,4 @@
+# Procedure macros for pyo3-polars
+
+This crate is not intended for direct usage. Use `pyo3-polars` to get the functionality
+of this crate.

pyo3-polars/Cargo.toml

@@ -1,27 +1,25 @@
 [package]
 name = "pyo3-polars"
-version = "0.6.0"
+version = "0.7.0"
 edition = "2021"
 license = "MIT"
 readme = "../README.md"
 repository = "https://github.com/pola-rs/pyo3-polars"
-description = "PyO3 bindings to polars"
+description = "Expression plugins and PyO3 types for polars"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+ciborium = { version = "0.2.1", optional = true }
 polars = { workspace = true, default-features = false }
 polars-core = { workspace = true, default-features = false }
 polars-ffi = { workspace = true, optional = true }
-polars-plan = { workspace = true, optional = true }
 polars-lazy = { workspace = true, optional = true }
+polars-plan = { workspace = true, optional = true }
 pyo3 = "0.19.0"
+pyo3-polars-derive = { version = "0.1.0", path = "../pyo3-polars-derive", optional = true }
 thiserror = "1"
-ciborium = { version = "0.2.1", optional = true }
-pyo3-polars-derive = { version = "*", path = "../pyo3-polars-derive", optional=true }
-
 
 [features]
 lazy = ["polars/serde-lazy", "polars-plan", "polars-lazy/serde", "ciborium"]
 derive = ["pyo3-polars-derive", "polars-plan", "polars-ffi"]
-