diff --git a/Cargo.lock b/Cargo.lock
index 3b98b5d8b..424c2fa00 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4576,6 +4576,7 @@ dependencies = [
  "k9",
  "log",
  "portable-pty",
+ "pretty_env_logger",
  "regex",
  "shell-words",
  "smol",
diff --git a/wezterm-ssh/Cargo.toml b/wezterm-ssh/Cargo.toml
index 8542d115a..d25c835da 100644
--- a/wezterm-ssh/Cargo.toml
+++ b/wezterm-ssh/Cargo.toml
@@ -19,6 +19,7 @@ ssh2 = "0.9"
 
 [dev-dependencies]
 k9 = "0.11.0"
+pretty_env_logger = "0.4"
 shell-words = "1.0"
 structopt = "0.3"
 termwiz = { path = "../termwiz" }
diff --git a/wezterm-ssh/examples/ssh.rs b/wezterm-ssh/examples/ssh.rs
index 300f73bac..821b6ace1 100644
--- a/wezterm-ssh/examples/ssh.rs
+++ b/wezterm-ssh/examples/ssh.rs
@@ -45,6 +45,7 @@ struct Opt {
 }
 
 fn main() {
+    pretty_env_logger::init();
     let opts = Opt::from_args();
 
     let mut config = Config::new();
diff --git a/wezterm-ssh/src/auth.rs b/wezterm-ssh/src/auth.rs
index d2cf583b1..b264b1516 100644
--- a/wezterm-ssh/src/auth.rs
+++ b/wezterm-ssh/src/auth.rs
@@ -2,6 +2,7 @@ use crate::session::SessionEvent;
 use anyhow::Context;
 use smol::channel::{bounded, Sender};
 use std::collections::HashSet;
+use std::path::{Path, PathBuf};
 
 #[derive(Debug)]
 pub struct AuthenticationPrompt {
@@ -28,6 +29,97 @@ impl AuthenticationEvent {
 }
 
 impl crate::session::SessionInner {
+    fn agent_auth(&mut self, sess: &ssh2::Session, user: &str) -> anyhow::Result<bool> {
+        if let Some(only) = self.config.get("identitiesonly") {
+            if only == "yes" {
+                return Ok(false);
+            }
+        }
+
+        let mut agent = sess.agent()?;
+        agent.connect()?;
+        agent.list_identities()?;
+        let identities = agent.identities()?;
+        for identity in identities {
+            if agent.userauth(user, &identity).is_ok() {
+                return Ok(true);
+            }
+        }
+        Ok(false)
+    }
+
+    fn pubkey_auth(
+        &mut self,
+        sess: &ssh2::Session,
+        user: &str,
+        host: &str,
+    ) -> anyhow::Result<bool> {
+        if let Some(files) = self.config.get("identityfile") {
+            for file in files.split_whitespace() {
+                let pubkey: PathBuf = format!("{}.pub", file).into();
+                let file = Path::new(file);
+
+                if !file.exists() {
+                    continue;
+                }
+
+                let pubkey = if pubkey.exists() && false {
+                    Some(pubkey.as_ref())
+                } else {
+                    None
+                };
+
+                match sess.userauth_pubkey_file(user, pubkey, &file, None) {
+                    Ok(_) => return Ok(true),
+                    Err(err) => {
+                        if err.code() == ssh2::ErrorCode::Session(-16)
+                            || err.code() == ssh2::ErrorCode::Session(-18)
+                        {
+                            // Need a passphrase to decrypt the key
+
+                            let (reply, answers) = bounded(1);
+                            self.tx_event
+                                .try_send(SessionEvent::Authenticate(AuthenticationEvent {
+                                    username: "".to_string(),
+                                    instructions: "".to_string(),
+                                    prompts: vec![AuthenticationPrompt {
+                                        prompt: format!(
+                                            "Passphrase to decrypt {} for {}@{}: ",
+                                            file.display(),
+                                            user,
+                                            host
+                                        ),
+                                        echo: false,
+                                    }],
+                                    reply,
+                                }))
+                                .context("sending Authenticate request to user")?;
+
+                            let answers = smol::block_on(answers.recv())
+                                .context("waiting for authentication answers from user")?;
+
+                            if answers.is_empty() {
+                                anyhow::bail!("user cancelled authentication");
+                            }
+
+                            let passphrase = &answers[0];
+
+                            match sess.userauth_pubkey_file(user, pubkey, &file, Some(passphrase)) {
+                                Ok(_) => return Ok(true),
+                                Err(err) => {
+                                    log::warn!("pubkey auth: {:#}", err);
+                                }
+                            }
+                        } else {
+                            log::warn!("pubkey auth: {:#}", err);
+                        }
+                    }
+                }
+            }
+        }
+        Ok(false)
+    }
+
     pub fn authenticate(
         &mut self,
         sess: &ssh2::Session,
@@ -46,10 +138,20 @@ impl crate::session::SessionInner {
             log::trace!("ssh auth methods: {:?}", methods);
 
             if !sess.authenticated() && methods.contains("publickey") {
-                if let Err(err) = sess.userauth_agent(&user) {
-                    log::warn!("while attempting agent auth: {}", err);
-                } else {
-                    continue;
+                match self.agent_auth(sess, user) {
+                    Ok(true) => continue,
+                    Ok(false) => {}
+                    Err(err) => {
+                        log::warn!("while attempting agent auth: {}", err)
+                    }
+                }
+
+                match self.pubkey_auth(sess, user, host) {
+                    Ok(true) => continue,
+                    Ok(false) => {}
+                    Err(err) => {
+                        log::warn!("while attempting auth: {}", err)
+                    }
                 }
             }
 
diff --git a/wezterm-ssh/src/config.rs b/wezterm-ssh/src/config.rs
index de31fcead..5dc9d3b6c 100644
--- a/wezterm-ssh/src/config.rs
+++ b/wezterm-ssh/src/config.rs
@@ -279,6 +279,18 @@ impl Config {
             }
         }
 
+        if !result.contains_key("identityfile") {
+            if let Some(home) = self.resolve_home() {
+                result.insert(
+                    "identityfile".to_string(),
+                    format!(
+                        "{}/.ssh/id_dsa {}/.ssh/id_ecdsa {}/.ssh/id_ed25519 {}/.ssh/id_rsa",
+                        home, home, home, home
+                    ),
+                );
+            }
+        }
+
         if !result.contains_key("identityagent") {
             if let Some(sock_path) = self.resolve_env("SSH_AUTH_SOCK") {
                 result.insert("identityagent".to_string(), sock_path);
diff --git a/wezterm-ssh/src/session.rs b/wezterm-ssh/src/session.rs
index 0eebbc905..f038eb01f 100644
--- a/wezterm-ssh/src/session.rs
+++ b/wezterm-ssh/src/session.rs
@@ -114,6 +114,7 @@ impl SessionInner {
             .context("setting TCP NODELAY on ssh connection")?;
 
         let mut sess = ssh2::Session::new()?;
+        // sess.trace(ssh2::TraceFlags::all());
         sess.set_tcp_stream(tcp);
         sess.handshake()
             .with_context(|| format!("ssh handshake with {}", remote_address))?;