windows: stub out a winpty wrapper

conpty, while it may be the best native API available for consoles
built in to Windows 10, currently swallows mouse escape sequences
(https://github.com/Microsoft/console/issues/376) so we need an
alternative for a better windows experience.

winpty appears to be a reasonably mature pty implementation, so
let's try that!

I tried to use winpty-sys from crates.io but it requires the
installation of an external clang compiler so I used bindgen
on my linux box to convert the small header file to rust and
then tweaked it a bit.

This commit includes that basic wrapper plus a type safe layer
on top.

This will require distributing the dll and an agent helper along
with the wezterm.exe.

This commit doesn't implement the wezterm side of the pty interface;
that will be in a follow up commit.

src/pty/conpty.rs

@@ -397,7 +397,7 @@ impl PsuedoCon {
 }
 
 #[derive(Debug)]
-struct OwnedHandle {
+pub struct OwnedHandle {
     handle: HANDLE,
 }
 unsafe impl Send for OwnedHandle {}
@@ -410,7 +410,10 @@ impl Drop for OwnedHandle {
 }
 
 impl OwnedHandle {
-    fn try_clone(&self) -> Result<Self, IoError> {
+    pub fn new(handle: HANDLE) -> Self {
+        Self { handle }
+    }
+    pub fn try_clone(&self) -> Result<Self, IoError> {
         if self.handle == INVALID_HANDLE_VALUE || self.handle.is_null() {
             return Ok(OwnedHandle {
                 handle: self.handle,

src/pty/mod.rs

@@ -2,6 +2,8 @@
 pub mod conpty;
 #[cfg(unix)]
 pub mod unix;
+#[cfg(windows)]
+pub mod winpty;
 
 #[cfg(windows)]
 pub use self::conpty::{openpty, Child, Command, ExitStatus, MasterPty, SlavePty};

src/pty/winpty/mod.rs

@@ -0,0 +1,2 @@
+mod safe;
+mod sys;

src/pty/winpty/safe.rs

@@ -0,0 +1,263 @@
+//! A type-safe wrapper around the sys module, which in turn exposes
+//! the API exported by winpty.dll.
+//! https://github.com/rprichard/winpty/blob/master/src/include/winpty.h
+use super::sys::*;
+use crate::pty::conpty::OwnedHandle;
+use bitflags::bitflags;
+use failure::{format_err, Error};
+use std::ffi::{OsStr, OsString};
+use std::os::windows::ffi::{OsStrExt, OsStringExt};
+use std::ptr;
+use winapi::shared::minwindef::DWORD;
+use winapi::shared::ntdef::LPCWSTR;
+use winapi::um::fileapi::{CreateFileW, OPEN_EXISTING};
+use winapi::um::handleapi::INVALID_HANDLE_VALUE;
+use winapi::um::winbase::INFINITE;
+use winapi::um::winnt::HANDLE;
+use winapi::um::winnt::{GENERIC_READ, GENERIC_WRITE};
+
+bitflags! {
+    pub struct AgentFlags : u64 {
+        const CONERR = WINPTY_FLAG_CONERR;
+        const PLAIN_OUTPUT = WINPTY_FLAG_PLAIN_OUTPUT;
+        const COLOR_ESCAPES = WINPTY_FLAG_COLOR_ESCAPES;
+        const ALLOW_DESKTOP_CREATE = WINPTY_FLAG_ALLOW_CURPROC_DESKTOP_CREATION;
+    }
+}
+bitflags! {
+    pub struct SpawnFlags : u64 {
+        const AUTO_SHUTDOWN = WINPTY_SPAWN_FLAG_AUTO_SHUTDOWN;
+        const EXIT_AFTER_SHUTDOWN = WINPTY_SPAWN_FLAG_EXIT_AFTER_SHUTDOWN;
+    }
+}
+
+#[repr(u32)]
+enum MouseMode {
+    None = WINPTY_MOUSE_MODE_NONE,
+    Auto = WINPTY_MOUSE_MODE_AUTO,
+    Force = WINPTY_MOUSE_MODE_FORCE,
+}
+
+enum Timeout {
+    Infinite,
+    Milliseconds(DWORD),
+}
+
+struct WinPtyConfig {
+    config: *mut winpty_config_t,
+}
+
+fn wstr_to_osstr(wstr: LPCWSTR) -> Result<OsString, Error> {
+    ensure!(!wstr.is_null(), "LPCWSTR is null");
+    let slice = unsafe { std::slice::from_raw_parts(wstr, libc::wcslen(wstr)) };
+    Ok(OsString::from_wide(slice))
+}
+
+fn wstr_to_string(wstr: LPCWSTR) -> Result<String, Error> {
+    ensure!(!wstr.is_null(), "LPCWSTR is null");
+    let slice = unsafe { std::slice::from_raw_parts(wstr, libc::wcslen(wstr)) };
+    String::from_utf16(slice).map_err(|e| format_err!("String::from_utf16: {}", e))
+}
+
+fn check_err<T>(err: winpty_error_ptr_t, value: T) -> Result<T, Error> {
+    ensure!(!err.is_null(), "winpty error object is null");
+    unsafe {
+        let code = (WINPTY.winpty_error_code)(err);
+        if code == WINPTY_ERROR_SUCCESS {
+            return Ok(value);
+        }
+
+        let converted = wstr_to_string((WINPTY.winpty_error_msg)(err))?;
+        (WINPTY.winpty_error_free)(err);
+        bail!("winpty error code {}: {}", code, converted)
+    }
+}
+
+impl WinPtyConfig {
+    pub fn new(flags: AgentFlags) -> Result<Self, Error> {
+        let mut err: winpty_error_ptr_t = ptr::null_mut();
+        let config = unsafe { (WINPTY.winpty_config_new)(flags.bits(), &mut err) };
+        let config = check_err(err, config)?;
+        ensure!(
+            !config.is_null(),
+            "winpty_config_new returned nullptr but no error"
+        );
+        Ok(Self { config })
+    }
+
+    pub fn set_initial_size(&mut self, cols: c_int, rows: c_int) {
+        unsafe { (WINPTY.winpty_config_set_initial_size)(self.config, cols, rows) }
+    }
+
+    pub fn set_mouse_mode(&mut self, mode: MouseMode) {
+        unsafe { (WINPTY.winpty_config_set_mouse_mode)(self.config, mode as c_int) }
+    }
+
+    pub fn set_agent_timeout(&mut self, timeout: Timeout) {
+        let duration = match timeout {
+            Timeout::Infinite => INFINITE,
+            Timeout::Milliseconds(n) => n,
+        };
+        unsafe { (WINPTY.winpty_config_set_agent_timeout)(self.config, duration) }
+    }
+
+    pub fn open(&self) -> Result<WinPty, Error> {
+        let mut err: winpty_error_ptr_t = ptr::null_mut();
+        let pty = unsafe { (WINPTY.winpty_open)(self.config, &mut err) };
+        let pty = check_err(err, pty)?;
+        ensure!(!pty.is_null(), "winpty_open returned nullptr but no error");
+        Ok(WinPty { pty })
+    }
+}
+
+impl Drop for WinPtyConfig {
+    fn drop(&mut self) {
+        unsafe { (WINPTY.winpty_config_free)(self.config) }
+    }
+}
+
+pub struct WinPty {
+    pty: *mut winpty_t,
+}
+
+impl Drop for WinPty {
+    fn drop(&mut self) {
+        unsafe { (WINPTY.winpty_free)(self.pty) }
+    }
+}
+
+fn pipe_client(name: LPCWSTR) -> Result<OwnedHandle, Error> {
+    let handle = unsafe {
+        CreateFileW(
+            name,
+            GENERIC_READ | GENERIC_WRITE,
+            0,
+            ptr::null_mut(),
+            OPEN_EXISTING,
+            0,
+            ptr::null_mut(),
+        )
+    };
+    if handle == INVALID_HANDLE_VALUE {
+        let err = std::io::Error::last_os_error();
+        bail!("failed to open {:?}: {}", wstr_to_string(name), err);
+    } else {
+        Ok(OwnedHandle::new(handle))
+    }
+}
+
+impl WinPty {
+    pub fn agent_process(&self) -> HANDLE {
+        unsafe { (WINPTY.winpty_agent_process)(self.pty) }
+    }
+
+    pub fn conin(&self) -> Result<OwnedHandle, Error> {
+        pipe_client(unsafe { (WINPTY.winpty_conin_name)(self.pty) })
+    }
+
+    pub fn conout(&self) -> Result<OwnedHandle, Error> {
+        pipe_client(unsafe { (WINPTY.winpty_conout_name)(self.pty) })
+    }
+
+    pub fn conerr(&self) -> Result<OwnedHandle, Error> {
+        pipe_client(unsafe { (WINPTY.winpty_conerr_name)(self.pty) })
+    }
+
+    pub fn set_size(&mut self, rows: c_int, cols: c_int) -> Result<bool, Error> {
+        let mut err: winpty_error_ptr_t = ptr::null_mut();
+        let result = unsafe { (WINPTY.winpty_set_size)(self.pty, rows, cols, &mut err) };
+        Ok(result != 0)
+    }
+
+    pub fn spawn(&mut self, config: &SpawnConfig) -> Result<SpawnedProcess, Error> {
+        let mut err: winpty_error_ptr_t = ptr::null_mut();
+        let mut create_process_error: DWORD = 0;
+        let mut process_handle: HANDLE = ptr::null_mut();
+        let mut thread_handle: HANDLE = ptr::null_mut();
+
+        let result = unsafe {
+            (WINPTY.winpty_spawn)(
+                self.pty,
+                config.spawn_config,
+                &mut process_handle,
+                &mut thread_handle,
+                &mut create_process_error,
+                &mut err,
+            )
+        };
+        let thread_handle = OwnedHandle::new(thread_handle);
+        let process_handle = OwnedHandle::new(process_handle);
+        let result = check_err(err, result)?;
+        if result == 0 {
+            let err = std::io::Error::from_raw_os_error(create_process_error as _);
+            bail!("winpty_spawn failed: {}", err);
+        }
+        Ok(SpawnedProcess {
+            thread_handle,
+            process_handle,
+        })
+    }
+}
+
+pub struct SpawnedProcess {
+    process_handle: OwnedHandle,
+    thread_handle: OwnedHandle,
+}
+
+pub struct SpawnConfig {
+    spawn_config: *mut winpty_spawn_config_t,
+}
+
+/// Construct a null terminated wide string from an OsStr
+fn str_to_wide(s: &OsStr) -> Vec<u16> {
+    let mut wide: Vec<u16> = s.encode_wide().collect();
+    wide.push(0);
+    wide
+}
+
+fn str_ptr(s: &Option<Vec<u16>>) -> LPCWSTR {
+    match s {
+        None => ptr::null(),
+        Some(v) => v.as_ptr(),
+    }
+}
+
+impl SpawnConfig {
+    pub fn new(
+        flags: SpawnFlags,
+        appname: Option<&OsStr>,
+        cmdline: Option<&OsStr>,
+        cwd: Option<&OsStr>,
+        env: Option<&OsStr>,
+    ) -> Result<Self, Error> {
+        let appname = appname.map(str_to_wide);
+        let cmdline = cmdline.map(str_to_wide);
+        let cwd = cwd.map(str_to_wide);
+        let env = env.map(str_to_wide);
+
+        let mut err: winpty_error_ptr_t = ptr::null_mut();
+
+        let spawn_config = unsafe {
+            (WINPTY.winpty_spawn_config_new)(
+                flags.bits(),
+                str_ptr(&appname),
+                str_ptr(&cmdline),
+                str_ptr(&cwd),
+                str_ptr(&env),
+                &mut err,
+            )
+        };
+        let spawn_config = check_err(err, spawn_config)?;
+        ensure!(
+            !spawn_config.is_null(),
+            "winpty_spawn_config_new returned nullptr but no error"
+        );
+        Ok(Self { spawn_config })
+    }
+}
+
+impl Drop for SpawnConfig {
+    fn drop(&mut self) {
+        unsafe { (WINPTY.winpty_spawn_config_free)(self.spawn_config) }
+    }
+}

src/pty/winpty/sys.rs

@@ -0,0 +1,108 @@
+//! A rust wrapper around winpty.dll
+//! https://github.com/rprichard/winpty/blob/master/src/include/winpty.h
+//! This was partially generated by bindgen and then tweaked to work
+//! with the shared_library macro.
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+#![allow(non_camel_case_types)]
+#![allow(non_upper_case_globals)]
+
+use lazy_static::lazy_static;
+use shared_library::shared_library;
+use std::path::Path;
+use winapi::shared::minwindef::{BOOL, DWORD};
+use winapi::shared::ntdef::LPCWSTR;
+use winapi::um::winnt::HANDLE;
+
+pub use ::std::os::raw::c_int;
+
+pub const WINPTY_ERROR_SUCCESS: u32 = 0;
+pub const WINPTY_ERROR_OUT_OF_MEMORY: u32 = 1;
+pub const WINPTY_ERROR_SPAWN_CREATE_PROCESS_FAILED: u32 = 2;
+pub const WINPTY_ERROR_LOST_CONNECTION: u32 = 3;
+pub const WINPTY_ERROR_AGENT_EXE_MISSING: u32 = 4;
+pub const WINPTY_ERROR_UNSPECIFIED: u32 = 5;
+pub const WINPTY_ERROR_AGENT_DIED: u32 = 6;
+pub const WINPTY_ERROR_AGENT_TIMEOUT: u32 = 7;
+pub const WINPTY_ERROR_AGENT_CREATION_FAILED: u32 = 8;
+pub const WINPTY_FLAG_CONERR: u64 = 1;
+pub const WINPTY_FLAG_PLAIN_OUTPUT: u64 = 2;
+pub const WINPTY_FLAG_COLOR_ESCAPES: u64 = 4;
+pub const WINPTY_FLAG_ALLOW_CURPROC_DESKTOP_CREATION: u64 = 8;
+pub const WINPTY_MOUSE_MODE_NONE: u32 = 0;
+pub const WINPTY_MOUSE_MODE_AUTO: u32 = 1;
+pub const WINPTY_MOUSE_MODE_FORCE: u32 = 2;
+pub const WINPTY_SPAWN_FLAG_AUTO_SHUTDOWN: u64 = 1;
+pub const WINPTY_SPAWN_FLAG_EXIT_AFTER_SHUTDOWN: u64 = 2;
+
+pub struct winpty_error_t {}
+pub struct winpty_t {}
+pub struct winpty_spawn_config_t {}
+pub struct winpty_config_t {}
+
+pub type winpty_error_ptr_t = *mut winpty_error_t;
+pub type winpty_result_t = DWORD;
+
+shared_library!(WinPtyFuncs,
+    pub fn winpty_error_code(err: winpty_error_ptr_t) -> winpty_result_t,
+    pub fn winpty_error_msg(err: winpty_error_ptr_t) -> LPCWSTR,
+    pub fn winpty_error_free(err: winpty_error_ptr_t),
+    pub fn winpty_config_new(
+        agentFlags: u64,
+        err: *mut winpty_error_ptr_t
+    ) -> *mut winpty_config_t,
+    pub fn winpty_config_free(cfg: *mut winpty_config_t),
+    pub fn winpty_config_set_initial_size(
+        cfg: *mut winpty_config_t,
+        cols: c_int,
+        rows: c_int
+    ),
+    pub fn winpty_config_set_mouse_mode(
+        cfg: *mut winpty_config_t,
+        mouseMode: c_int
+    ),
+    pub fn winpty_config_set_agent_timeout(
+        cfg: *mut winpty_config_t,
+        timeoutMs: DWORD
+    ),
+    pub fn winpty_open(cfg: *const winpty_config_t, err: *mut winpty_error_ptr_t) -> *mut winpty_t,
+    pub fn winpty_agent_process(wp: *mut winpty_t) -> HANDLE,
+    pub fn winpty_conin_name(wp: *mut winpty_t) -> LPCWSTR,
+    pub fn winpty_conout_name(wp: *mut winpty_t) -> LPCWSTR,
+    pub fn winpty_conerr_name(wp: *mut winpty_t) -> LPCWSTR,
+    pub fn winpty_spawn_config_new(
+        spawnFlags: u64,
+        appname: LPCWSTR,
+        cmdline: LPCWSTR,
+        cwd: LPCWSTR,
+        env: LPCWSTR,
+        err: *mut winpty_error_ptr_t
+    ) -> *mut winpty_spawn_config_t,
+    pub fn winpty_spawn_config_free(cfg: *mut winpty_spawn_config_t),
+    pub fn winpty_spawn(
+        wp: *mut winpty_t,
+        cfg: *const winpty_spawn_config_t,
+        process_handle: *mut HANDLE,
+        thread_handle: *mut HANDLE,
+        create_process_error: *mut DWORD,
+        err: *mut winpty_error_ptr_t
+    ) -> BOOL,
+    pub fn winpty_set_size(
+        wp: *mut winpty_t,
+        cols: c_int,
+        rows: c_int,
+        err: *mut winpty_error_ptr_t
+    ) -> BOOL,
+    pub fn winpty_get_console_process_list(
+        wp: *mut winpty_t,
+        processList: *mut c_int,
+        processCount: c_int,
+        err: *mut winpty_error_ptr_t
+    ) -> c_int,
+    pub fn winpty_free(wp: *mut winpty_t),
+);
+
+lazy_static! {
+    pub static ref WINPTY: WinPtyFuncs =
+        WinPtyFuncs::open(Path::new("winpty.dll")).expect("winpty.dll is required");
+}