How to binding a custom iterator?

[starwing]

I'm trying to binding my API to Lua, but I have meet troubles when binding iterator, this is a minimual test for that:

fn test_iter() -> LuaResult<()> {
    use std::collections::BTreeSet;

    let lua = Lua::new();
    #[derive(Default)]
    struct MySet(BTreeSet<i32>);

    impl Deref for MySet {
        type Target = BTreeSet<i32>;

        fn deref(&self) -> &Self::Target {
            &self.0
        }
    }

    impl DerefMut for MySet {
        fn deref_mut(&mut self) -> &mut Self::Target {
            &mut self.0
        }
    }

    struct MyIter<'a>(std::collections::btree_set::Iter<'a, i32>);

    impl<'a> Iterator for MyIter<'a> {
        type Item = i32;

        fn next(&mut self) -> Option<Self::Item> {
            self.0.next().cloned()
        }
    }

    impl<'a> LuaUserData for MyIter<'a> {
        fn add_methods<'lua, M: LuaUserDataMethods<'lua, Self>>(methods: &mut M) {
            methods.add_meta_method_mut(LuaMetaMethod::Call, |_, iter, ()| Ok(iter.next()));
        }
    }

    impl LuaUserData for MySet {
        fn add_methods<'lua, M: LuaUserDataMethods<'lua, Self>>(methods: &mut M) {
            methods.add_method("iter", |lua, set, ()| {
                let iter = MyIter::<'lua>(set.iter());
                Ok(LuaValue::UserData(lua.create_userdata(iter)?)) // <--- error here
            });
        }
    }

    let mut set = MySet::default();
    set.insert(1);
    set.insert(2);
    set.insert(3);
    lua.globals().set("set", set)?;
    lua.load(
        r#"
        print(set)
        for v in set:iter() do
            print(v)
        end
        "#,
    )
    .exec()
}

it errors:

error[E0521]: borrowed data escapes outside of closure
  --> main.rs:57:39
   |
55 |             methods.add_method("iter", |lua, set, ()| {
   |                                              ---
   |                                              |
   |                                              `set` is a reference that is only valid in the closure body
   |                                              has type `&'1 MySet`
56 |                 let iter = MyIter::<'lua>(set.iter());
57 |                 Ok(LuaValue::UserData(lua.create_userdata(iter)?))
   |                                       ^^^^^^^^^^^^^^^^^^^^^^^^^
   |                                       |
   |                                       `set` escapes the closure body here
   |                                       argument requires that `'1` must outlive `'static`

error: lifetime may not live long enough
  --> main.rs:57:39
   |
54 |         fn add_methods<'lua, M: LuaUserDataMethods<'lua, Self>>(methods: &mut M) {
   |                        ---- lifetime `'lua` defined here
...
57 |                 Ok(LuaValue::UserData(lua.create_userdata(iter)?))
   |                                       ^^^^^^^^^^^^^^^^^^^^^^^^^ argument requires that `'lua` must outlive `'static`

For more information about this error, try `rustc --explain E0521`.

So how could I deal with it?

[KKould]

I have the same question, how to deal with the iterator lifetime

[zerodegress]

Try this.
If you are single thread program, use Rc<RefCell>,if not,use Arc<Mutex>.
I suggest always let script engines run on single thread to reduce unnecessary hassles.

fn test_iter() -> mlua::Result<()> {
    use std::collections::BTreeSet;

    let lua = Lua::new();
    #[derive(Default, Clone)]
    struct MySet(Rc<RefCell<BTreeSet<i32>>>);

    impl Deref for MySet {
        type Target = RefCell<BTreeSet<i32>>;

        fn deref(&self) -> &Self::Target {
            &self.0
        }
    }

    struct MyIter{
        set: MySet,
        cur_index: usize,
    }

    impl MyIter {
        fn new(set: MySet) -> Self {
            MyIter {
                set,
                cur_index: 0,
            }
        }
    }

    impl Iterator for MyIter {
        type Item = i32;

        fn next(&mut self) -> Option<Self::Item> {
            let res = (*self.set).borrow().iter().nth(self.cur_index).map(|x| *x);
            self.cur_index += 1;
            res
        }
    }

    impl<'a> UserData for MyIter {
        fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
            methods.add_meta_method_mut(MetaMethod::Call, |_, iter, ()| Ok(iter.next()));
        }
    }

    impl UserData for MySet {
        fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
            methods.add_method("iter", |lua, set, ()| {
                let iter = MyIter::new(set.clone());
                Ok(Value::UserData(lua.create_userdata(iter)?)) // <--- error here
            });
        }
    }

    let mut set = MySet::default();
    set.borrow_mut().insert(1);
    set.borrow_mut().insert(2);
    set.borrow_mut().insert(3);
    lua.globals().set("set", set)?;
    lua.load(
        r#"
        print(set)
        for v in set:iter() do
            print(v)
        end
        "#,
    )
    .exec()
}

[starwing]

@zerodegress it makes iterating a BTreeSet O(n^2*log(n)) complexity.

[zerodegress]

@zerodegress it makes iterating a BTreeSet O(n^2*log(n)) complexity.

Unfortunately,you cannot predict that the set behind the iter is written as they are read,so you have to repeatedly iterate it to make sure sequentially read the iter while the set owned by other.
Better solution is to copy the whole set when iterate it if you can afford the costs of copy.

[khvzak]

Sorry for the delayed response. There are few ways how to bind a custom iterator, eg. using scoped userdata, or tricks with self-referential types.

An example of using a self-referential struct that holds iterator to BTreeSet (any other container would work too):

use std::collections::{btree_set, BTreeSet};
use std::rc::Rc;

use mlua::{AnyUserData, Function, Lua, Result, UserDataMethods, UserDataRefMut};
use ouroboros::self_referencing;

#[self_referencing]
struct BTreeSetIter {
    set: Rc<BTreeSet<String>>,
    #[borrows(set)]
    #[covariant]
    iter: btree_set::Iter<'this, String>,
}

fn main() -> Result<()> {
    let lua = Lua::new();

    lua.register_userdata_type::<Rc<BTreeSet<String>>>(|reg| {
        reg.add_method("iter", |_, this, ()| {
            let next = Function::wrap(|lua, mut it: UserDataRefMut<BTreeSetIter>| {
                it.with_iter_mut(|iter| match iter.next() {
                    Some(value) => Ok(Some(lua.create_string(value)?)),
                    None => Ok(None),
                })
            });
            let iter_ud = AnyUserData::wrap(
                BTreeSetIterBuilder {
                    set: this.clone(),
                    iter_builder: |set| set.iter(),
                }
                .build(),
            );
            Ok((next, iter_ud))
        });
    })?;

    let my_set = Rc::new(
        ["foo", "bar", "baz"]
            .iter()
            .map(ToString::to_string)
            .collect::<BTreeSet<_>>(),
    );
    lua.globals().set("my_set", AnyUserData::wrap(my_set))?;

    lua.load(
        r#"
            for value in my_set:iter() do
                print(value)
            end
        "#,
    )
    .exec()
}

It prints

bar
baz
foo

when called and has the same complexity as iterating directly from Rust.

[starwing]

Thanks for the reply!

Could you possibly provide some examples of how scoped userdata is used?

[khvzak]

After running few experiments I can say thay putting iterator to a scope to create nonstatic userdata is not the best idea / not ergonomic. Basically you will need to have an extra function with own scope to obtain a valid iterator that can be used as non-static userdata.

[starwing]

Okay, thanks for explain! :-)

[sxyazi]

Hey @khvzak, thanks for your answer. I've been watching this issue for a while, and the code above is very close to what I want to achieve.

However, my data is scoped userdata, not a string. Simply changing lua.create_string() to lua.create_any_userdata() doesn't make it work. So my question is, is there any way to create userdata within the register?

Currently, I'm using a clumsy method where I create all the needed userdata at once during the initialization of the scope, then store them using set_named_user_value, and later retrieve them in register_userdata_type. However, this limits its use case - I can't apply it to a larger amount of data - the complete data.

[khvzak]

@sxyazi would be possible to provide an example please, that I can use to run/fix?

In your code, is the window method problematic? Am I right that you want to return iterable object from window method?

[sxyazi]

@khvzak Yeah of course. Here it is:

use mlua::{AnyUserData, IntoLua, Lua, MetaMethod, UserDataFields, UserDataMethods, Variadic};

const LUA_CODE: &str = r#"
  for _, file in pairs(folder) do
    print(file.name)
  end
"#;

struct Folder {
  files: Vec<File>,
}

#[derive(Clone)]
struct File {
  name: String,
  // ...other fields
}

fn main() -> anyhow::Result<()> {
  let lua = Lua::new();
  let folder = Folder { files: (0..100).map(|i| File { name: i.to_string() }).collect() };

  lua.register_userdata_type::<Folder>(|reg| {
    reg.add_meta_function(MetaMethod::Pairs, |lua, me: AnyUserData| {
      let iter = lua.create_function(|lua, (me, i): (AnyUserData, usize)| {
        let folder = me.borrow::<Folder>()?;
        let i = i + 1;
        Ok(if i > folder.files.len() {
          Variadic::new()
        } else {
          Variadic::from_iter([
            i.into_lua(lua)?,
            // This doesn't work.
            // lua.create_any_userdata_ref(&folder.files[i - 1]),

            // This `clone()` is not necessary, what I want is to return a reference to the file,
            // but I don't know how to do that. Since the `Folder` and `File` are not native types,
            // I don't want to add `Rc` or `Arc` to them.
            lua.create_any_userdata(folder.files[i - 1].clone())?.into_lua(lua)?,
          ])
        })
      })?;
      Ok((iter, me, 0))
    });
  })?;

  lua.register_userdata_type::<File>(|reg| {
    reg.add_field_method_get("name", |lua, me| lua.create_string(&me.name));
  })?;

  Ok(lua.scope(|scope| {
    lua.globals().set("folder", scope.create_any_userdata_ref(&folder)?)?;

    lua.load(LUA_CODE).exec()?;

    Ok(())
  })?)
}

And this is my current approach: initializing all the necessary file data at once in the scope:

use mlua::{Lua, UserDataFields, Value};

const LUA_CODE: &str = r#"
  for _, file in ipairs(folder.window) do
    print(file.name)
  end
"#;

struct Folder {
  files: Vec<File>,
}

#[derive(Clone)]
struct File {
  name: String,
  // ...other fields
}

fn main() -> anyhow::Result<()> {
  let lua = Lua::new();
  let folder = Folder { files: (0..100).map(|i| File { name: i.to_string() }).collect() };

  lua.register_userdata_type::<Folder>(|reg| {
    reg.add_field_function_get("window", |_, me| me.named_user_value::<Value>("partial_files"));
  })?;

  lua.register_userdata_type::<File>(|reg| {
    reg.add_field_method_get("name", |lua, me| lua.create_string(&me.name));
  })?;

  Ok(lua.scope(|scope| {
    let partial_files: Vec<_> = folder
      .files
      .iter()
      .take(30)
      .map(|file| scope.create_any_userdata_ref(file))
      .filter_map(|result| result.ok())
      .collect();

    let folder = scope.create_any_userdata_ref(&folder)?;
    folder.set_named_user_value("partial_files", partial_files)?;
    lua.globals().set("folder", folder)?;

    lua.load(LUA_CODE).exec()?;

    Ok(())
  })?)
}

However, this limits the use cases - I cannot initialize a large number (let's say 100,000) of files via create_any_userdata_ref(), it's very slow.

But by using an iterator, when the conditions are met, I can break in Lua to avoid the subsequent create_any_userdata_ref()s overhead.

[sxyazi]

I want to add a create_any_userdata_ref to Lua, but after reading some of the source code, and it seems difficult to achieve without causing breaking changes, because register_userdata_type cannot access the scope's lifecycle.

@khvzak Would you accept adding an unsafe create_any_userdata_ref? We can use a different name to avoid confusion, so I only need to ensure the data's safety myself. If this is acceptable, I am willing to create a PR for it :)

[khvzak]

The Scope::create_any_userdata_ref is already exist, maybe you meant something else?

[khvzak]

so I only need to ensure the data's safety myself.

If you want to go to unsafe territory, you probably don't need a scope too.

Your example can be easily solved with just one unsafe:

struct PointerCell<T>(*const T);

impl<T> Deref for PointerCell<T> {
    type Target = T;

    fn deref(&self) -> &Self::Target {
        unsafe { &*self.0 }
    }
}

fn main() -> anyhow::Result<()> {
    let lua = Lua::new();
    let folder = Folder {
        files: (0..100)
            .map(|i| File {
                name: i.to_string(),
            })
            .collect(),
    };

    lua.register_userdata_type::<PointerCell<Folder>>(|reg| {
        reg.add_meta_function(MetaMethod::Pairs, |lua, me: AnyUserData| {
            let iter = lua.create_function(
                |lua, (folder, i): (UserDataRef<PointerCell<Folder>>, usize)| {
                    let i = i + 1;
                    if i > folder.files.len() {
                        ().into_lua_multi(lua)
                    } else {
                        (i, AnyUserData::wrap(PointerCell(&folder.files[i - 1])))
                            .into_lua_multi(lua)
                    }
                },
            )?;
            Ok((iter, me, 0))
        });
    })?;

    lua.register_userdata_type::<PointerCell<File>>(|reg| {
        reg.add_field_method_get("name", |lua, me| lua.create_string(&me.name));
    })?;

    lua.globals()
        .set("folder", AnyUserData::wrap(PointerCell(&folder)))?;

    lua.load(LUA_CODE).exec()?;

    Ok(())
}

It would work fine, you just need to ensure that reference to a &File will never be used after drop