diff --git a/Cargo.toml b/Cargo.toml
index e6e6c72d2..a733d6904 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,3 @@
-[workspace]
-
 [package]
 authors = ["Wez Furlong <wez@wezfurlong.org>"]
 name = "wezterm"
@@ -13,7 +11,10 @@ harfbuzz-sys = "0.1.15"
 libc = "0.2.36"
 mio = "0.6.12"
 resize = "0.3.0"
+serde = "1.0.27"
+serde_derive = "1.0.27"
 servo-fontconfig = "0.4.0"
+toml = "0.4.5"
 unicode-width = "0.1.4"
 
 [dependencies.term]
@@ -36,3 +37,5 @@ version = "0.2.0"
 
 [features]
 debug-escape-sequences = ["term/debug-escape-sequences"]
+
+[workspace]
diff --git a/README.md b/README.md
index 19a99545d..1fb1ccc1d 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ These are in the done/doing soon category:
 - [x] Runs on Linux with XCB
 - [x] Scrollback (use mouse wheel and Shift Page{Up|Down})
 - [x] True Color support
-- [x] Color Emoji and font fallback (Note: currently assumes you have the Operator font!)
+- [x] Color Emoji and font fallback
 - [x] Paste selection via Shift-Insert
 - [ ] xterm style selection of text with mouse
 - [ ] Configuration file to specify fonts and colors
@@ -58,3 +58,21 @@ Things that I'd like to see happen and that have no immediate priority
 - [ ] Tabs
 - [ ] Textual renderer.  Think `tmux` or `screen`.
 - [ ] Runs on Windows
+
+## Configuration
+
+`wezterm` will look for a TOML configuration file in `$HOME/.config/wezterm/wezterm.toml`,
+and then in `$HOME/.wezterm.toml`.
+
+Configuration is currently very simple and the format is considered unstable and subject
+to change.  The code for configuration can be found in `src/config.rs`.
+
+I use the following in my `~/.wezterm.toml`:
+
+```
+font_size = 10
+font = { fontconfig_pattern = "Operator Mono SSm Lig" }
+```
+
+The default configuration will attempt to use whichever font is returned from
+fontconfig when `monospace` is requested.
diff --git a/src/config.rs b/src/config.rs
new file mode 100644
index 000000000..93f9a8c2d
--- /dev/null
+++ b/src/config.rs
@@ -0,0 +1,94 @@
+//! Configuration for the gui portion of the terminal
+
+use failure::Error;
+use std;
+use std::fs;
+use std::io::prelude::*;
+use toml;
+
+
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    /// The font size, measured in points
+    #[serde(default = "default_font_size")]
+    pub font_size: f64,
+
+    /// The DPI to assume
+    #[serde(default = "default_dpi")]
+    pub dpi: f64,
+
+    #[serde(default)]
+    pub font: TextStyle,
+}
+
+fn default_font_size() -> f64 {
+    10.0
+}
+
+fn default_dpi() -> f64 {
+    96.0
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            font_size: default_font_size(),
+            dpi: default_dpi(),
+            font: TextStyle::default(),
+        }
+    }
+}
+
+/// Represents textual styling.
+/// TODO: I want to add some rules so that a user can specify the font
+/// and colors to use in some situations.  For example, xterm has
+/// a bold color option; I'd like to be able to express something
+/// like "when text is bold, use this font pattern and set the text
+/// color to X".  There are some interesting possibilities here;
+/// instead of just setting the color to a specific value we could
+/// apply a transform to the color attribute and make it X% brighter.
+#[derive(Debug, Deserialize)]
+pub struct TextStyle {
+    /// A font config pattern to parse to locate the font.
+    /// Note that the dpi and current font_size for the terminal
+    /// will be set on the parsed result.
+    pub fontconfig_pattern: String,
+}
+
+impl Default for TextStyle {
+    fn default() -> Self {
+        Self {
+            fontconfig_pattern: "monospace".into()
+        }
+    }
+}
+
+impl Config {
+    pub fn load() -> Result<Self, Error> {
+        let home = std::env::home_dir().ok_or_else(|| format_err!("can't find home dir"))?;
+
+        let paths = [
+            home.join(".config").join("wezterm").join("wezterm.toml"),
+            home.join(".wezterm.toml"),
+        ];
+
+        for p in paths.iter() {
+            let mut file = match fs::File::open(p) {
+                Ok(file) => file,
+                Err(err) => match err.kind() {
+                    std::io::ErrorKind::NotFound => continue,
+                    _ => bail!("Error opening {}: {:?}", p.display(), err),
+                }
+            };
+
+            let mut s = String::new();
+            file.read_to_string(&mut s)?;
+
+            return toml::from_str(&s).map_err(|e| {
+                    format_err!("Error parsing TOML from {}: {:?}", p.display(), e)
+            });
+        }
+
+        Ok(Self::default())
+    }
+}
diff --git a/src/main.rs b/src/main.rs
index 5942add78..6b1f2309a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -9,7 +9,11 @@ extern crate freetype;
 extern crate resize;
 extern crate libc;
 extern crate mio;
+extern crate serde;
+#[macro_use]
+extern crate serde_derive;
 extern crate term;
+extern crate toml;
 #[macro_use]
 pub mod log;
 
@@ -27,6 +31,7 @@ use std::process::Command;
 use std::str;
 use std::time::Duration;
 
+mod config;
 mod xgfx;
 mod xkeysyms;
 mod font;
@@ -61,11 +66,16 @@ fn run() -> Result<(), Error> {
 
     let waiter = sigchld::ChildWaiter::new()?;
 
+    let config = config::Config::load()?;
+    println!("Using configuration: {:#?}", config);
+
     // First step is to figure out the font metrics so that we know how
     // big things are going to be.
 
-    let mut pattern = FontPattern::parse("Operator Mono SSm Lig:size=10")?;
-    pattern.add_double("dpi", 96.0)?;
+    let mut pattern = FontPattern::parse(&config.font.fontconfig_pattern)?;
+    pattern.add_double("size", config.font_size)?;
+    pattern.add_double("dpi", config.dpi)?;
+
     let mut font = Font::new(pattern)?;
     // we always load the cell_height for font 0,
     // regardless of which font we are shaping here,