mononoke: allow using ALPN to identify hgcli traffic

Summary:
I'd like to eventually add an actual HTTP stack in Mononoke Server. Doing this
will be easier / nicer if we don't have to peek at the traffic to decide how
to interpret it (i.e. ALPN vs. HTTP).

To do so, we can use ALPN. If the client tells us they want the hgcli protocol,
we can use that, and skip peeking at the traffic entirely.

Obviously, we can't roll this out in one go, so first, let's accept traffic
advertised as hgcli via ALPN. Later, we can stop peeking at the traffic
entirely once we're not receiving any traffic from hgcli not advertised via
ALPN. I added ODS counters so we can canary this.

Note that the way I set this up is that if the client requests something that
isn't ALPN (e.g. H2), we just continue the handshake but don't select anything.

Finally, note that client side, we don't care (or even try to read) what the
server actually selected.

Reviewed By: johansglock

Differential Revision: D25954535

fbshipit-source-id: 183f6f56b2c8895aa8b18101565a4f2cd643be8d

eden/mononoke/hgcli/Cargo.toml

@@ -7,6 +7,7 @@ license = "GPLv2+"
 include = ["src/**/*.rs"]
 
 [dependencies]
+alpn = { path = "../alpn" }
 permission_checker = { path = "../permission_checker" }
 scuba_ext = { path = "../common/scuba_ext" }
 session_id = { path = "../server/session_id" }

eden/mononoke/hgcli/src/serve/mod.rs

@@ -325,6 +325,8 @@ impl<'a> StdioRelay<'a> {
             connector.set_certificate(&pkcs12.cert)?;
             connector.set_private_key(&pkcs12.pkey)?;
 
+            connector.set_alpn_protos(&alpn::alpn_format(alpn::HGCLI_ALPN)?)?;
+
             // add root certificate
 
             connector

eden/mononoke/server/Cargo.toml

@@ -7,6 +7,7 @@ license = "GPLv2+"
 include = ["src/**/*.rs"]
 
 [dependencies]
+alpn = { path = "../alpn" }
 cmdlib = { path = "../cmdlib" }
 monitoring = { path = "monitoring" }
 repo_listener = { path = "repo_listener" }

eden/mononoke/server/repo_listener/Cargo.toml

@@ -7,6 +7,7 @@ license = "GPLv2+"
 include = ["src/**/*.rs"]
 
 [dependencies]
+alpn = { path = "../../alpn" }
 backsyncer = { path = "../../commit_rewriting/backsyncer" }
 blobrepo = { path = "../../blobrepo" }
 blobrepo_factory = { path = "../../blobrepo/factory" }

eden/mononoke/server/repo_listener/src/connection_acceptor.rs

@@ -50,14 +50,21 @@ use limits::types::MononokeThrottleLimits;
 use sshrelay::{
     IoStream, Metadata, Preamble, Priority, SshDecoder, SshEncoder, SshEnvVars, SshMsg, Stdio,
 };
+use stats::prelude::*;
 
 use crate::errors::ErrorKind;
-use crate::repo_handlers::RepoHandler;
-use crate::request_handler::{create_conn_logger, request_handler};
-
 use crate::netspeedtest::{
     create_http_header, handle_http_netspeedtest, parse_netspeedtest_http_params, NetSpeedTest,
 };
+use crate::repo_handlers::RepoHandler;
+use crate::request_handler::{create_conn_logger, request_handler};
+
+define_stats! {
+    prefix = "mononoke.connection_acceptor";
+    http_accepted: timeseries(Sum),
+    hgcli_accepted: timeseries(Sum),
+    hgcli_no_alpn_accepted: timeseries(Sum),
+}
 
 #[cfg(fbcode_build)]
 const HEADER_ENCODED_CLIENT_IDENTITY: &str = "x-fb-validated-client-encoded-identity";
@@ -246,22 +253,35 @@ async fn server_mux(
     tls_identities: &MononokeIdentitySet,
     logger: &Logger,
 ) -> Result<MuxOutcome> {
+    let is_hgcli = s.ssl().selected_alpn_protocol() == Some(alpn::HGCLI_ALPN.as_bytes());
+
     let is_trusted = security_checker.check_if_trusted(&tls_identities).await?;
     let (mut rx, mut tx) = tokio::io::split(s);
 
     // Elaborate scheme to workaround lack of peek() on AsyncRead
     let mut peek_buf = vec![0; 4];
     rx.read_exact(&mut peek_buf[..]).await?;
-    let is_http = match peek_buf.as_slice() {
+    let is_http = if is_hgcli {
-        // For non-HTTP connection this can never start with GET or POST as these
+        STATS::hgcli_accepted.add_value(1);
-        // are wrapped in NetString encoding and prefixed with a type, so
+        false
-        // should start with:
+    } else {
-        // <number>:\x00
+        match peek_buf.as_slice() {
-        //
+            // For non-HTTP connection this can never start with GET or POST as these
-        // For example:
+            // are wrapped in NetString encoding and prefixed with a type, so
-        // 7:\x00hello\n,
+            // should start with:
-        b"GET " | b"POST" => true,
+            // <number>:\x00
-        _ => false,
+            //
+            // For example:
+            // 7:\x00hello\n,
+            b"GET " | b"POST" => {
+                STATS::http_accepted.add_value(1);
+                true
+            }
+            _ => {
+                STATS::hgcli_no_alpn_accepted.add_value(1);
+                false
+            }
+        }
     };
     let buf_rx = std::io::Cursor::new(peek_buf).chain(BufReader::new(rx));
 

eden/mononoke/server/src/main.rs

@@ -8,16 +8,14 @@
 #![deny(warnings)]
 #![feature(never_type)]
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use cloned::cloned;
 use cmdlib::{args, monitoring::ReadyFlagService};
 use fbinit::FacebookInit;
 use futures::channel::oneshot;
+use openssl::ssl::AlpnError;
 use slog::{error, info};
 
-#[cfg(fbcode_build)]
-use openssl as _; // suppress unused crate warning - only used outside fbcode
-
 fn setup_app<'a, 'b>() -> args::MononokeClapApp<'a, 'b> {
     let app = args::MononokeAppBuilder::new("mononoke server")
         .with_shutdown_timeout_args()
@@ -60,14 +58,23 @@ fn main(fb: FacebookInit) -> Result<()> {
         let private_key = matches.value_of("private_key").unwrap().to_string();
         let ca_pem = matches.value_of("ca_pem").unwrap().to_string();
 
-        secure_utils::SslConfig::new(
+        let mut builder = secure_utils::SslConfig::new(
             ca_pem,
             cert,
             private_key,
             matches.value_of("ssl-ticket-seeds"),
         )
-        .build_tls_acceptor(root_log.clone())
+        .tls_acceptor_builder(root_log.clone())
-        .expect("failed to build tls acceptor")
+        .context("Failed to instantiate TLS Acceptor builder")?;
+
+        builder.set_alpn_select_callback(|_, protos| {
+            // NOTE: Currently we do not support HTTP/2 here yet.
+            alpn::alpn_select(protos, alpn::HGCLI_ALPN)
+                .map_err(|_| AlpnError::ALERT_FATAL)?
+                .ok_or(AlpnError::NOACK)
+        });
+
+        builder.build()
     };
 
     info!(root_log, "Creating repo listeners");