config: make wezterm.action more ergonomic

you can try this in the debug overlay repl:

```
> wezterm.action{QuickSelectArgs={}}
{
    "QuickSelectArgs": {},
}
```

```
> wezterm.action.QuickSelectArgs
{
    "QuickSelectArgs": {
        "alphabet": "",
        "label": "",
        "patterns": {},
    },
}
```

```
> wezterm.action.QuickSelectArgs{alphabet="abc"}
{
    "QuickSelectArgs": {
        "alphabet": "abc",
    },
}
```

```
> wezterm.action.QuickSelectArgs{}
{
    "QuickSelectArgs": {},
}
```

```
> wezterm.action.Copy
"Copy"
```

```
> wezterm.action.ActivatePaneByIndex(1)
{
    "ActivatePaneByIndex": 1,
}
```

refs: https://github.com/wez/wezterm/issues/1150

config/src/lua.rs

@@ -149,7 +149,8 @@ pub fn make_lua_context(config_file: &Path) -> anyhow::Result<Lua> {
             lua.create_function(font_with_fallback)?,
         )?;
         wezterm_mod.set("hostname", lua.create_function(hostname)?)?;
-        wezterm_mod.set("action", lua.create_function(action)?)?;
+        wezterm_mod.set("action", luahelper::enumctor::Enum::<KeyAssignment>::new())?;
+
         lua.set_named_registry_value(LUA_REGISTRY_USER_CALLBACK_COUNT, 0)?;
         wezterm_mod.set("action_callback", lua.create_function(action_callback)?)?;
 
@@ -402,22 +403,6 @@ fn font_with_fallback<'lua>(
     Ok(text_style)
 }
 
-/// Helper for defining key assignment actions.
-/// Usage looks like this:
-///
-/// ```lua
-/// local wezterm = require 'wezterm';
-/// return {
-///    keys = {
-///      {key="{", mods="SHIFT|CTRL", action=wezterm.action{ActivateTabRelative=-1}},
-///      {key="}", mods="SHIFT|CTRL", action=wezterm.action{ActivateTabRelative=1}},
-///    }
-/// }
-/// ```
-fn action<'lua>(lua: &'lua Lua, action: Table<'lua>) -> mlua::Result<KeyAssignment> {
-    Ok(KeyAssignment::from_lua(Value::Table(action), lua)?)
-}
-
 fn action_callback<'lua>(lua: &'lua Lua, callback: mlua::Function) -> mlua::Result<KeyAssignment> {
     let callback_count: i32 = lua.named_registry_value(LUA_REGISTRY_USER_CALLBACK_COUNT)?;
     let user_event_id = format!("user-defined-{}", callback_count);

luahelper/src/enumctor.rs

@@ -0,0 +1,193 @@
+use crate::{dynamic_to_lua_value, lua_value_to_dynamic};
+use mlua::{Lua, MetaMethod, ToLua, UserData, UserDataMethods, Value};
+use std::collections::BTreeMap;
+use std::marker::PhantomData;
+use wezterm_dynamic::{
+    Error as DynError, FromDynamic, FromDynamicOptions, ToDynamic, UnknownFieldAction,
+    Value as DynValue,
+};
+
+struct EnumVariant<T> {
+    phantom: PhantomData<T>,
+    variant: String,
+}
+
+// Safety: <T> is used only in PhantomData so it doesn't actually
+// need to be Send.
+unsafe impl<T> Send for EnumVariant<T> {}
+
+impl<T> EnumVariant<T>
+where
+    T: FromDynamic,
+    T: ToDynamic,
+    T: std::fmt::Debug,
+    T: 'static,
+{
+    fn new(variant: String) -> Self {
+        Self {
+            phantom: PhantomData,
+            variant,
+        }
+    }
+
+    fn call_impl<'lua>(variant: &str, lua: &'lua Lua, table: Value) -> mlua::Result<Value<'lua>> {
+        let value = lua_value_to_dynamic(table)?;
+
+        let mut obj = BTreeMap::new();
+        obj.insert(DynValue::String(variant.to_string()), value);
+        let value = DynValue::Object(obj.into());
+
+        let _action = T::from_dynamic(
+            &value,
+            FromDynamicOptions {
+                unknown_fields: UnknownFieldAction::Deny,
+                deprecated_fields: UnknownFieldAction::Warn,
+            },
+        )
+        .map_err(|e| mlua::Error::external(e.to_string()))?;
+        dynamic_to_lua_value(lua, value)
+    }
+}
+
+impl<T> UserData for EnumVariant<T>
+where
+    T: FromDynamic,
+    T: ToDynamic,
+    T: std::fmt::Debug,
+    T: 'static,
+{
+    fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
+        methods.add_meta_method(MetaMethod::Call, |lua, myself, table: Value| {
+            Self::call_impl(&myself.variant, lua, table)
+        });
+    }
+}
+
+/// This type is used as an enum constructor for type `T`.
+/// The primary usage is to enable `wezterm.action` to have the following
+/// behaviors for KeyAssignment:
+///
+/// `wezterm.action{QuickSelectArgs={}}` -> compatibility with prior versions;
+/// the table is passed through and from_dynamic -> lua conversion is attempted.
+///
+/// `wezterm.action.QuickSelectArgs` -> since the `QuickSelectArgs` variant
+/// has a payload that impl Default, this is equivalent to the call above.
+///
+/// `wezterm.action.QuickSelectArgs{}` -> equivalent to the call above, but
+/// explicitly calls the constructor with no parameters.
+///
+/// `wezterm.action.QuickSelectArgs{alphabet="abc"}` -> configures the alphabet.
+///
+/// This dynamic behavior is implemented using metatables.
+///
+/// `Enum<T>` implements __call to handle that first case above, and __index
+/// to handle the remaining cases.
+///
+/// The __index implementation will return a simple string value for unit variants,
+/// which is how they are encoded by to_dynamic.
+///
+/// Otherwise, a table will be built with the equivalent value representation.
+/// That table will also have a metatable assigned to it, which allows for
+/// either using the value as-is or passing additional parameters to it.
+///
+/// If parameters are required, an EnumVariant<T> is returned instead of the
+/// table, and it has a __call method that will perform that final stage
+/// of construction.
+pub struct Enum<T> {
+    phantom: PhantomData<T>,
+}
+
+// Safety: <T> is used only in PhantomData so it doesn't actually
+// need to be Send.
+unsafe impl<T> Send for Enum<T> {}
+
+impl<T> Enum<T> {
+    pub fn new() -> Self {
+        Self {
+            phantom: PhantomData,
+        }
+    }
+}
+
+impl<T> UserData for Enum<T>
+where
+    T: FromDynamic,
+    T: ToDynamic,
+    T: std::fmt::Debug,
+    T: 'static,
+{
+    fn add_methods<'lua, M: UserDataMethods<'lua, Self>>(methods: &mut M) {
+        methods.add_meta_method(MetaMethod::Call, |lua, _myself, table: Value| {
+            let value = lua_value_to_dynamic(table)?;
+            let _action = T::from_dynamic(
+                &value,
+                FromDynamicOptions {
+                    unknown_fields: UnknownFieldAction::Deny,
+                    deprecated_fields: UnknownFieldAction::Warn,
+                },
+            )
+            .map_err(|e| mlua::Error::external(e.to_string()))?;
+            dynamic_to_lua_value(lua, value)
+        });
+
+        methods.add_meta_method(MetaMethod::Index, |lua, _myself, field: String| {
+            // Step 1: see if this is a unit variant.
+            // A unit variant will be convertible from string
+            if let Ok(_) = T::from_dynamic(
+                &DynValue::String(field.to_string()),
+                FromDynamicOptions {
+                    unknown_fields: UnknownFieldAction::Deny,
+                    deprecated_fields: UnknownFieldAction::Ignore,
+                },
+            ) {
+                return Ok(field.to_lua(lua)?);
+            }
+
+            // Step 2: see if this is a valid variant, and whether we can
+            // default-construct it with an empty table.
+            let mut obj = BTreeMap::new();
+            obj.insert(
+                DynValue::String(field.to_string()),
+                DynValue::Object(BTreeMap::<DynValue, DynValue>::new().into()),
+            );
+            match T::from_dynamic(
+                &DynValue::Object(obj.into()),
+                FromDynamicOptions {
+                    unknown_fields: UnknownFieldAction::Deny,
+                    deprecated_fields: UnknownFieldAction::Ignore,
+                },
+            ) {
+                Ok(defaulted) => {
+                    let defaulted = defaulted.to_dynamic();
+                    match dynamic_to_lua_value(lua, defaulted)? {
+                        Value::Table(t) => {
+                            let mt = lua.create_table()?;
+                            mt.set(
+                                "__call",
+                                lua.create_function(move |lua, (_mt, table): (Value, Value)| {
+                                    EnumVariant::<T>::call_impl(&field, lua, table)
+                                })?,
+                            )?;
+
+                            t.set_metatable(Some(mt));
+                            return Ok(Value::Table(t));
+                        }
+                        wat => Err(mlua::Error::external(format!(
+                            "unexpected type {}",
+                            wat.type_name()
+                        ))),
+                    }
+                }
+                err @ Err(DynError::InvalidVariantForType { .. }) => {
+                    Err(mlua::Error::external(err.unwrap_err().to_string()))
+                }
+                _ => {
+                    // Must be a valid variant, but requires arguments
+                    let variant_ctor =
+                        lua.create_userdata(EnumVariant::<T>::new(field.to_string()))?;
+                    Ok(Value::UserData(variant_ctor))
+                }
+            }
+        });
+    }
+}

luahelper/src/lib.rs

@@ -7,6 +7,8 @@ use std::collections::{BTreeMap, HashSet};
 use std::rc::Rc;
 use wezterm_dynamic::{FromDynamic, ToDynamic, Value as DynValue};
 
+pub mod enumctor;
+
 /// Implement lua conversion traits for a type.
 /// This implementation requires that the type implement
 /// FromDynamic and ToDynamic.