From 33b8a6aa623e8288e44a4f39c5bedc8beb7b0a8b Mon Sep 17 00:00:00 2001 From: Stanislau Hlebik Date: Mon, 20 Nov 2017 05:22:09 -0800 Subject: [PATCH] mononoke: HTTPS in eden server Summary: Avoid using plain HTTP and use HTTPS instead. To do this config needs to provide paths to server certificate and private key files in PEM format. Then they will be converted to Pkcs12 archive. This diff adds authentication of server i.e. client can check that it talks to a real server. Next diff adds authentication of a client. Lower-level `hyper::server::Http::bind_connection()` is used instead of `hyper::server::Http::bind()` method in order to add TLS support. See code comments for more details. Implementation is more complicated than I expected it to be. I need to use 3 more new crates. Lmk if there is a better way to do this. Reviewed By: jsgf Differential Revision: D6323440 fbshipit-source-id: 544f27e6ec210ddf840212b0c0c94145980e8be3 --- eden_server/src/errors.rs | 10 +++- eden_server/src/main.rs | 69 +++++++++++++++++++++++----- tests/integration/edenservertest.crt | 21 +++++++++ tests/integration/edenservertest.key | 28 +++++++++++ tests/integration/test-eden-server.t | 49 ++++++++++---------- 5 files changed, 140 insertions(+), 37 deletions(-) create mode 100644 tests/integration/edenservertest.crt create mode 100644 tests/integration/edenservertest.key diff --git a/eden_server/src/errors.rs b/eden_server/src/errors.rs index c1f1eddaae..c4138b25dc 100644 --- a/eden_server/src/errors.rs +++ b/eden_server/src/errors.rs @@ -4,18 +4,24 @@ // This software may be used and distributed according to the terms of the // GNU General Public License version 2 or any later version. +use std::string::ParseError; + use blobrepo; use mercurial_types; -use std::string::ParseError; +use native_tls; +use secure_utils; #[recursion_limit = "1024"] error_chain! { links { - MercurialTypes(mercurial_types::Error, mercurial_types::ErrorKind); Blobrepo(blobrepo::Error, blobrepo::ErrorKind); + MercurialTypes(mercurial_types::Error, mercurial_types::ErrorKind); + SecureUtils(secure_utils::Error, secure_utils::ErrorKind); } foreign_links { + IoError(::std::io::Error); + NativeTlsError(native_tls::Error); // This error is an Rust awkward hack // https://doc.rust-lang.org/std/string/enum.ParseError.html. Should never be encountered ParseError(ParseError); diff --git a/eden_server/src/main.rs b/eden_server/src/main.rs index a04f8c507c..3bcae6e7e4 100644 --- a/eden_server/src/main.rs +++ b/eden_server/src/main.rs @@ -26,7 +26,9 @@ extern crate hyper; #[macro_use] extern crate lazy_static; extern crate mercurial_types; +extern crate native_tls; extern crate regex; +extern crate secure_utils; extern crate serde; #[macro_use] extern crate serde_derive; @@ -35,6 +37,7 @@ extern crate serde_json; extern crate slog; extern crate slog_glog_fmt; extern crate tokio_core; +extern crate tokio_tls; extern crate toml; use std::collections::HashMap; @@ -43,10 +46,11 @@ use std::ffi::OsString; use std::fs::File; use std::io::Read; use std::os::unix::ffi::OsStringExt; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::str::FromStr; use std::string::ToString; use std::sync::Arc; +use tokio_core::net::TcpListener; use tokio_core::reactor::Core; use blobrepo::{BlobRepo, BlobState, FilesBlobState, RocksBlobState, TestManifoldBlobState}; @@ -59,8 +63,10 @@ use futures_ext::{BoxFuture, FutureExt, StreamExt}; use hyper::StatusCode; use hyper::server::{Http, Request, Response, Service}; use mercurial_types::{NodeHash, Repo}; +use native_tls::TlsAcceptor; use regex::{Captures, Regex}; use slog::{Drain, Level, Logger}; +use tokio_tls::TlsAcceptorExt; mod errors; @@ -308,26 +314,57 @@ where } } -fn start_server(addr: &str, reponame: String, state: State, logger: Logger) -where +fn start_server( + addr: &str, + reponame: String, + state: State, + logger: Logger, + cert: P, + private_key: P, +) where State: BlobState, + P: AsRef, { let addr = addr.parse().expect("Failed to parse address"); let mut map = HashMap::new(); let repo = BlobRepo::new(state); map.insert(reponame, Arc::new(repo)); + let pkcs12 = + secure_utils::build_pkcs12(cert, private_key).expect("cannot build pkcs12 archive"); + // Acceptor that has `accept_async()` method that handles tls handshake + // and returns decrypted stream. + let tlsacceptor = TlsAcceptor::builder(pkcs12) + .expect("cannot create TlsAcceptorBuilder") + .build() + .expect("cannot create TlsAcceptor"); + + let mut core = Core::new().expect("cannot create http server core"); + let handle = core.handle(); + let listener = TcpListener::bind(&addr, &handle).expect("cannot bind to the address"); + let incoming = listener.incoming().from_err::(); + info!(logger, "started eden server"); let cpupool = Arc::new(CpuPool::new_num_cpus()); - let func = move || { - Ok(EdenServer::new( - map.clone(), - cpupool.clone(), - logger.clone(), - )) - }; - let server = Http::new().bind(&addr, func).expect("Failed to run server"); - server.run().expect("Error while running service"); + let http_server = Http::new(); + let conns = incoming.for_each(|stream_addr| { + let (tcp_stream, remote_addr) = stream_addr; + let http_server = http_server.clone(); + let handle = handle.clone(); + let service = EdenServer::new(map.clone(), cpupool.clone(), logger.clone()); + let logger = logger.clone(); + tlsacceptor.accept_async(tcp_stream).then(move |stream| { + match stream { + Ok(stream) => { + http_server.bind_connection(&handle, stream, remote_addr, service); + } + Err(err) => error!(logger, "accept async failed {}", err), + }; + Ok(()) + }) + }); + + core.run(conns).expect("http server main loop failed"); } /// Types of repositories supported @@ -345,6 +382,8 @@ struct RawRepoConfig { repotype: RawRepoType, reponame: String, addr: String, + cert: String, + private_key: String, } fn main() { @@ -386,6 +425,8 @@ fn main() { FilesBlobState::new(&config.path.expect("Please specify a path to the blobrepo")) .expect("couldn't open blob state"), root_logger.clone(), + config.cert, + config.private_key, ), RawRepoType::BlobRocks => start_server( &config.addr, @@ -393,6 +434,8 @@ fn main() { RocksBlobState::new(&config.path.expect("Please specify a path to the blobrepo")) .expect("couldn't open blob state"), root_logger.clone(), + config.cert, + config.private_key, ), RawRepoType::BlobManifold => { let (sender, receiver) = oneshot::channel(); @@ -421,6 +464,8 @@ fn main() { &remote, ).expect("couldn't open blob state"), root_logger.clone(), + config.cert, + config.private_key, ) } }; diff --git a/tests/integration/edenservertest.crt b/tests/integration/edenservertest.crt new file mode 100644 index 0000000000..063c42751c --- /dev/null +++ b/tests/integration/edenservertest.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDfzCCAmegAwIBAgIJAJ9JCSIKw+gmMA0GCSqGSIb3DQEBCwUAMFYxCzAJBgNV +BAYTAnVrMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg +Q29tcGFueSBMdGQxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xNzExMTQyMTIzMjRa +Fw0xODExMTQyMTIzMjRaMFYxCzAJBgNVBAYTAnVrMRUwEwYDVQQHDAxEZWZhdWx0 +IENpdHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxEjAQBgNVBAMMCWxv +Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALg4esLQCvsM +Dl7Oqjll5YtPLFbhkk0Xuq3+mv/zBfNMKUG6oeTRYsxVFzNCEypnxjgKL3P+AwkF +1Qi+BqApWapwdF7zfenPgtHSB3KmM485iEq2O8Z1mnQkpEYAXeRrOi/PmJuJvOuW +VtailcLcCZxVxclCxJuzRGGrYfr6/gnrk5dlWNa18RNkOAmxwmSK4/HqGkN/sgrD +L5gm/R05duhQtv9o6BCNqrrnQanpDc4EJEhFsoulW30Z/KyGT3eYoG6bTYZckhoN +tFrxWczv9KmtLFAnWknW4+bmKLJ4jXm/sTmngoXBnAJAlK/V8hln6HbJXawWNDXk +HwJZZ8myua8CAwEAAaNQME4wHQYDVR0OBBYEFJnLi3c8ekawXg4Rj80bfqhCovz6 +MB8GA1UdIwQYMBaAFJnLi3c8ekawXg4Rj80bfqhCovz6MAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggEBACGh4DBS0SabKNNCwLrE9fNrHpngKw46YUaprTo0 +l7w2+jbQ0EvcmW48pwSHhRaM3ZZb5dml31OvU28XnlGxSgJ+JBvIZ39qAeHw1eYR +AB6PHy+mtHsl+NOIadBc8AIgmG1D5LZ8TktqKAcqRf5cWi1QAyaBFlQO9C9IMfpS +qaaGKfLxdC6a4Tg4+Vq/DFmYW9mK9QFyJy2VejkrzNp7XltGqeoAhNtfFvUfKPci +xSUIHrPNK6V+gmFDpHc2E+5MgvHMIDbw7Ic4YK1//9BuHMOv7/9miVXztFD3UEkx +RXEu+AreAEGBOf5f2rizkzKNAeh7QfItcv/YFLYOfi4Y1jU= +-----END CERTIFICATE----- diff --git a/tests/integration/edenservertest.key b/tests/integration/edenservertest.key new file mode 100644 index 0000000000..df2393986a --- /dev/null +++ b/tests/integration/edenservertest.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC4OHrC0Ar7DA5e +zqo5ZeWLTyxW4ZJNF7qt/pr/8wXzTClBuqHk0WLMVRczQhMqZ8Y4Ci9z/gMJBdUI +vgagKVmqcHRe833pz4LR0gdypjOPOYhKtjvGdZp0JKRGAF3kazovz5ibibzrllbW +opXC3AmcVcXJQsSbs0Rhq2H6+v4J65OXZVjWtfETZDgJscJkiuPx6hpDf7IKwy+Y +Jv0dOXboULb/aOgQjaq650Gp6Q3OBCRIRbKLpVt9Gfyshk93mKBum02GXJIaDbRa +8VnM7/SprSxQJ1pJ1uPm5iiyeI15v7E5p4KFwZwCQJSv1fIZZ+h2yV2sFjQ15B8C +WWfJsrmvAgMBAAECggEAIA5T7kpdXsn+RikYHdzJULB6OrQNzTRv248OUbNsOaXr +F/Dt8u8sjfnQi67Xvu1H5MCA+WIeDRfith668GvBmpLu7QbZxjHBxdkSSrT5C9cr +DDUhuasFMWie3T97FyBBg8hP+eDB6wzCF587CClZ0sZqIFdp9t3+7C/yYMYEJp13 +oo2G2l0Hv1DW6XXUbeM1ic7lND+5J8EFhi/WLjpK7kvL9ImRaQmJnwIB007GXaxo +YeYiS3TFejhErZeYlfVJt8PIghlMFl1rjxTzW1dYakrp8FbOn6Madc1+1iWFdTp2 +YwBv7FIu4D4yCLlZ+6cZDn6BeDX+BjiXbeEPFZB5AQKBgQDvvZFdqP2ZCOI/KUXY +islvXDMuxx+7muvbN96WdRyud2OM0jB4W3+JlMcFatZ3drHWSlO26rNlBtMcD3Zl +JuPxg6nktFtiz32jGCCerqUy5vpwf5IT30cQ3nfIrJq/XpKOBwZqsC/HalIKPIfZ +G4681BxRdz625gQxPi7ah1zyoQKBgQDEtvZXIQ/YQfyUWLpCqYuArx3BTWRrt+Vl +/km9WbZkk/lqdgIoWmNIjktROnrMp+S2AqXVhU9TmZ27Zb5VlUQAESlBLemFJBj1 +afQk+lBrV8UL1lAoXtmWT/9JZzGuQff9F7e+vhYdF4DxMqr5ALKLI7fp2C/5q6ye +sWN1fq+aTwKBgQCtsZCyZQ9nYvJYhJSgMN2Emy5SA97P04xyRWY1aCAhn0o26spH +STN8AJi5KhC8ePibozpH2n+jAHDPcbx7yNN5VcUveCSF0ILcOUzY6vuxh3uHKkYx +s6EPbb8nWza609AmqpxRi9jyB+vskrJx7+9Yi+0AcP61EK2U59KWbWBAQQKBgAVR +nR+F+VOzlQy9yWt8AhWcpoVPfmObOUykr4G7Jz9Y8Ol7/1rv9rdDga3UkvVtpLV3 +JNy9GgP8p1Ml0RunSYtm62KjxyiebT5VexB05C/C08UaKlitF90ElLZ7X9CCYB4r +wgAaS6bFt+rNn+nTpwA0Gwjrm4B4r9YqnbHTlxPpAoGAQu5XjML3g1lrXqz6J2Zi +A326GQQxIkl4mhrBT1G8O8q6ugeIi9PrEn7QlOJvhb24FpRiYEfynhn8kn8EXqUd +qfcIWz3Jdf6yluh3lUR/YO5DnOG6Adoq/DS1LeKV4W4RGtvw0JAiQV0IkQmBIrf1 +icA0XxRpcb4QTpuq5KqLJpM= +-----END PRIVATE KEY----- diff --git a/tests/integration/test-eden-server.t b/tests/integration/test-eden-server.t index 5759fc4260..9823db9a6b 100644 --- a/tests/integration/test-eden-server.t +++ b/tests/integration/test-eden-server.t @@ -86,6 +86,8 @@ Add commit with a directory $ echo 'reponame="repo"' >> $TESTTMP/config $ echo "path=\"$TESTTMP/blobrepo\"" >> $TESTTMP/config $ echo 'addr="127.0.0.1:3000"' >> $TESTTMP/config + $ echo "cert=\"$TESTDIR/edenservertest.crt\"" >> $TESTTMP/config + $ echo "private_key=\"$TESTDIR/edenservertest.key\"" >> $TESTTMP/config #if files $ blobimport --blobstore files --linknodes repo $TESTTMP/blobrepo -d 2> out.txt @@ -136,14 +138,15 @@ Temporary hack to make sure server is ready $ sleep 1 Curl and debugdata output should match - $ curl http://localhost:3000/repo/cs/3903775176ed42b1458a6281db4a0ccf4d9f287a/roottreemanifestid 2> /dev/null + $ alias curl="curl --cacert $TESTDIR/edenservertest.crt" + $ curl https://localhost:3000/repo/cs/3903775176ed42b1458a6281db4a0ccf4d9f287a/roottreemanifestid 2> /dev/null 8515d4bfda768e04af4c13a69a72e28c7effbea7 (no-eol) $ cd repo $ hg debugdata -c 3903775176ed42b1458a6281db4a0ccf4d9f287a | head -n 1 8515d4bfda768e04af4c13a69a72e28c7effbea7 $ hg debugdata -m 8515d4bfda768e04af4c13a69a72e28c7effbea7 a\x00b80de5d138758541c5f05265ad144ab9fa86d1db (esc) - $ curl http://localhost:3000/repo/cs/533267b0e203537fa53d2aec834b062f0b2249cd/roottreemanifestid 2> /dev/null + $ curl https://localhost:3000/repo/cs/533267b0e203537fa53d2aec834b062f0b2249cd/roottreemanifestid 2> /dev/null 47827ecc7f12d2ed0c387de75947e73cf1c53afe (no-eol) $ hg debugdata -m 47827ecc7f12d2ed0c387de75947e73cf1c53afe @@ -152,7 +155,7 @@ Curl and debugdata output should match c\x005d9299349fc01ddd25d0070d149b124d8f10411e (esc) d\x00fc702583f9c961dea176fd367862c299b4a551f2 (esc) - $ curl http://localhost:3000/repo/treenode/8515d4bfda768e04af4c13a69a72e28c7effbea7/ 2> /dev/null| jq 'sort_by(.path)' + $ curl https://localhost:3000/repo/treenode/8515d4bfda768e04af4c13a69a72e28c7effbea7/ 2> /dev/null| jq 'sort_by(.path)' [ { "hash": "b80de5d138758541c5f05265ad144ab9fa86d1db", @@ -163,11 +166,11 @@ Curl and debugdata output should match ] Empty file - $ curl http://localhost:3000/repo/blob/b80de5d138758541c5f05265ad144ab9fa86d1db/ 2> /dev/null + $ curl https://localhost:3000/repo/blob/b80de5d138758541c5f05265ad144ab9fa86d1db/ 2> /dev/null - $ curl http://localhost:3000/repo/cs/4dabaf45f54add88ca2797dfdeb00a7d55144243/roottreemanifestid 2> /dev/null + $ curl https://localhost:3000/repo/cs/4dabaf45f54add88ca2797dfdeb00a7d55144243/roottreemanifestid 2> /dev/null b47dc781a873595c796b01e2ed5829e3fed2c887 (no-eol) - $ curl http://localhost:3000/repo/treenode/b47dc781a873595c796b01e2ed5829e3fed2c887/ 2> /dev/null| jq 'sort_by(.path)' + $ curl https://localhost:3000/repo/treenode/b47dc781a873595c796b01e2ed5829e3fed2c887/ 2> /dev/null| jq 'sort_by(.path)' [ { "hash": "b80de5d138758541c5f05265ad144ab9fa86d1db", @@ -188,9 +191,9 @@ Empty file "type": "File" } ] - $ curl http://localhost:3000/repo/blob/5d9299349fc01ddd25d0070d149b124d8f10411e/ 2> /dev/null + $ curl https://localhost:3000/repo/blob/5d9299349fc01ddd25d0070d149b124d8f10411e/ 2> /dev/null 2 - $ curl http://localhost:3000/repo/treenode/47827ecc7f12d2ed0c387de75947e73cf1c53afe/ 2> /dev/null | jq 'sort_by(.path)' + $ curl https://localhost:3000/repo/treenode/47827ecc7f12d2ed0c387de75947e73cf1c53afe/ 2> /dev/null | jq 'sort_by(.path)' [ { "hash": "b80de5d138758541c5f05265ad144ab9fa86d1db", @@ -217,16 +220,16 @@ Empty file "type": "File" } ] - $ curl http://localhost:3000/repo/blob/fc702583f9c961dea176fd367862c299b4a551f2/ 2> /dev/null + $ curl https://localhost:3000/repo/blob/fc702583f9c961dea176fd367862c299b4a551f2/ 2> /dev/null 2 - $ curl http://localhost:3000/repo/cs/617e87e2aa2fe36508e8d5e15a162bcd2e79808e/roottreemanifestid 2> /dev/null + $ curl https://localhost:3000/repo/cs/617e87e2aa2fe36508e8d5e15a162bcd2e79808e/roottreemanifestid 2> /dev/null ed8f515856d818e78bd52edac84a97568de65e0f (no-eol) - $ curl http://localhost:3000/repo/cs/617e87e2aa2fe36508e8d5e15a162bcd2e79808e/roottreemanifestid/ 2> /dev/null + $ curl https://localhost:3000/repo/cs/617e87e2aa2fe36508e8d5e15a162bcd2e79808e/roottreemanifestid/ 2> /dev/null ed8f515856d818e78bd52edac84a97568de65e0f (no-eol) - $ curl http://localhost:3000/repo/treenode/ed8f515856d818e78bd52edac84a97568de65e0f/ 2> /dev/null | jq 'sort_by(.path)' + $ curl https://localhost:3000/repo/treenode/ed8f515856d818e78bd52edac84a97568de65e0f/ 2> /dev/null | jq 'sort_by(.path)' [ { "hash": "e7405b0462d8b2dd80219b713a93aea2c9a3c468", @@ -236,7 +239,7 @@ Empty file } ] - $ curl http://localhost:3000/repo/treenode/e7405b0462d8b2dd80219b713a93aea2c9a3c468/ 2> /dev/null | jq 'sort_by(.path)' + $ curl https://localhost:3000/repo/treenode/e7405b0462d8b2dd80219b713a93aea2c9a3c468/ 2> /dev/null | jq 'sort_by(.path)' [ { "hash": "7108421418404a937c684d2479a34a24d2ce4757", @@ -245,7 +248,7 @@ Empty file "type": "File" } ] - $ curl http://localhost:3000/repo/treenode/e7405b0462d8b2dd80219b713a93aea2c9a3c468 2> /dev/null | jq 'sort_by(.path)' + $ curl https://localhost:3000/repo/treenode/e7405b0462d8b2dd80219b713a93aea2c9a3c468 2> /dev/null | jq 'sort_by(.path)' [ { "hash": "7108421418404a937c684d2479a34a24d2ce4757", @@ -255,25 +258,25 @@ Empty file } ] - $ curl http://localhost:3000/repo/blob/7108421418404a937c684d2479a34a24d2ce4757/ 2> /dev/null + $ curl https://localhost:3000/repo/blob/7108421418404a937c684d2479a34a24d2ce4757/ 2> /dev/null content - $ curl http://localhost:3000/repo/blob/7108421418404a937c684d2479a34a24d2ce4757 2> /dev/null + $ curl https://localhost:3000/repo/blob/7108421418404a937c684d2479a34a24d2ce4757 2> /dev/null content Send incorrect requests - $ curl http://localhost:3000/repo/cs/hash/roottreemanifestid 2> /dev/null + $ curl https://localhost:3000/repo/cs/hash/roottreemanifestid 2> /dev/null invalid sha-1 input: need at least 40 hex digits (no-eol) - $ curl http://localhost:3000/badrepo/cs/3903775176ed42b1458a6281db4a0ccf4d9f287a/roottreemanifestid 2> /dev/null + $ curl https://localhost:3000/badrepo/cs/3903775176ed42b1458a6281db4a0ccf4d9f287a/roottreemanifestid 2> /dev/null Error: unknown repo - $ curl http://localhost:3000/badrepo/treenode/8515d4bfda768e04af4c13a69a72e28c7effbea7/ 2> /dev/null + $ curl https://localhost:3000/badrepo/treenode/8515d4bfda768e04af4c13a69a72e28c7effbea7/ 2> /dev/null Error: unknown repo - $ curl http://localhost:3000/repo/BADURL/3903775176ed42b1458a6281db4a0ccf4d9f287a/roottreemanifestid 2> /dev/null + $ curl https://localhost:3000/repo/BADURL/3903775176ed42b1458a6281db4a0ccf4d9f287a/roottreemanifestid 2> /dev/null malformed url (no-eol) - $ curl http://localhost:3000/repo/cs/3903775176ed42b1458a6281db4a0ccf4d9f287a/roottreemanifestid/more 2> /dev/null + $ curl https://localhost:3000/repo/cs/3903775176ed42b1458a6281db4a0ccf4d9f287a/roottreemanifestid/more 2> /dev/null malformed url (no-eol) - $ curl http://localhost:3000/repo/cs/ 2> /dev/null + $ curl https://localhost:3000/repo/cs/ 2> /dev/null malformed url (no-eol) - $ curl http://localhost:3000/ 2> /dev/null + $ curl https://localhost:3000/ 2> /dev/null malformed url (no-eol) Make sure there are no errors on the server