on master help sign in
Merge remote-tracking branch 'origin/https-deploy' into routes
commit7bd1b2a11e846bb909c95a60a1e0af1e0d2e670b authorRyan Trinkle <ryan.trinkle@gmail.com> dateSat 30 Jun 2018, 21:02 (8 years ago) parenta8484423b7c664
10 files changed+208−66
README.md+15−9 default.nix+24−14 lib/command/obelisk-command.cabal+2 lib/command/src/Obelisk/Command.hs+11−1 lib/command/src/Obelisk/Command/Deploy.hs+121−24 lib/command/src/Obelisk/Command/Nix.hs+20−5 lib/https/default.nix−7 lib/https/github.json−6 migration/07900e6c41ab9a78917f66fcf7e0d9b8-8c84f81d9b09a5e0f028f3c6341ffb36/obelisk-handoff migration/07900e6c41ab9a78917f66fcf7e0d9b8-8c84f81d9b09a5e0f028f3c6341ffb36/obelisk-upgrade+15
README.md+15−9
@@ -97,22 +97,28 @@ First create a new EC2 instance:
97971. Launch a NixOS 17.09 EC2 instance (we recommend [this AMI](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#LaunchInstanceWizard:ami=ami-40bee63a))
98981. In the instance configuration wizard ensure that your instance has at least 1GB RAM and 10GB disk space.
99991. When prompted save your AWS private key (`~/myaws.pem`) somewhere safe. We'll need it later during deployment.
100−1. Go to "Security Groups", select your instance's security group and under "Inbound" tab add a new rule for HTTP port 80.
100+1. Go to "Security Groups", select your instance's security group and under "Inbound" tab add a new rule for HTTP port 80 and 443.
101101
102−At this stage your instance should be booting and become accessible shortly. Note down the hostname of your instance. It should look like this:
103−
104−```
105−INSTANCE_HOSTNAME=ec2-??-??-??-??.ca-central-1.compute.amazonaws.com
106−```
102+At this stage your instance should be booting and become accessible shortly. Note down the hostname of your EC2 instance.
107103
108104Now go to your Obelisk project directory (`~/code/myapp`), and initialize a deployment config (`~/code/myapp-deploy`):
109105Your project directory must be "thunkable", i.e. something on which `ob thunk pack` can be called. Usually it will be a git repository whose current revision has been pushed upstream.
110106
111107```
112108cd ~/code/myapp
113−ob deploy init --ssh-key ~/myaws.pem --hostname ${INSTANCE_HOSTNAME} ~/code/myapp-deploy
109+SERVER=ec2-35-183-22-197.ca-central-1.compute.amazonaws.com
110+ROUTE=https://myapp.com # Publicly accessible route to your app
111+EMAIL=myname@myapp.com
112+ob deploy init \
113+ --ssh-key ~/myaws.pem \
114+ --hostname $SERVER \
115+ --route $ROUTE \
116+ --admin-email $EMAIL \
117+ ~/code/myapp-deploy
114118```
115119
120+NOTE: HTTPS is enabled by default; to disable https, pass `--disable-https` to the `ob deploy init` command above.
121+
116122Then go to that created deployment configuration directory, and initiate the deployment:
117123
118124```
@@ -120,9 +126,9 @@ cd ~/code/myapp-deploy
120126ob deploy push
121127```
122128
123−`ob deploy push` will locally build your app and then transfer it, along with all the Nix package dependencies, via ssh to the EC2 instance. It will also configure Nginx so that the public port 80 proxies to the running app.
129+`ob deploy push` will locally build your app and then transfer it, along with all the Nix package dependencies, via ssh to the EC2 instance.
124130
125−At this point you are done. Your app will be accessible at `http://${HOSTNAME}`!
131+At this point you are done. Your app will be accessible at `${ROUTE}`.
126132
127133### Deploying an updated version
128134
default.nix+24−14
@@ -193,25 +193,33 @@ rec {
193193 ln -s ${compressedJs frontend} $out/frontend.jsexe
194194 ''; #TODO: run frontend.jsexe through the asset processing pipeline
195195
196− server = exe: hostName:
196+ server = { exe, hostName, adminEmail, routeHost, enableHttps }:
197197 let system = "x86_64-linux";
198198 nixos = import (pkgs.path + /nixos);
199− https = (import lib/https {}).module {
200− backendPort = 8000; # TODO read from config
201− # sslConfig = {
202− # hostName = "example.com";
203− # adminEmail = "webmaster@example.com";
204− # subdomains = [ "www" ];
205− # };
206− };
199+ backendPort = 8000;
207200 in nixos {
208201 inherit system;
209202 configuration = args: {
210203 imports = [
211204 (pkgs.path + /nixos/modules/virtualisation/amazon-image.nix)
212− https
213205 ];
214− networking = { inherit hostName; };
206+ networking = {
207+ inherit hostName;
208+ firewall.allowedTCPPorts = if enableHttps then [ 80 443 ] else [ 80 ];
209+ };
210+ services.nginx = {
211+ enable = true;
212+ virtualHosts."${routeHost}" = {
213+ enableACME = enableHttps;
214+ forceSSL = enableHttps;
215+ locations."/" = {
216+ proxyPass = "http://localhost:" + toString backendPort;
217+ };
218+ };
219+ };
220+ security.acme.certs = if enableHttps then {
221+ "${routeHost}".email = adminEmail;
222+ } else { };
215223 systemd.services.backend = {
216224 wantedBy = [ "multi-user.target" ];
217225 after = [ "network.target" ];
@@ -298,10 +306,12 @@ rec {
298306 };
299307 };
300308 in mkProject (projectDefinition args));
301− exe = serverExe (projectOut "x86_64-linux").ghc.backend (projectOut system).ghcjs.frontend assets.symlinked configPath;
309+ serverOn = sys: serverExe (projectOut sys).ghc.backend (projectOut system).ghcjs.frontend assets.symlinked configPath;
310+ linuxserver = serverOn "x86_64-linux";
302311 in projectOut system // {
303− inherit exe;
304− server = { hostName }: server exe hostName;
312+ inherit linuxserver;
313+ server = args@{ hostName, adminEmail, routeHost, enableHttps }:
314+ server (args // { exe = linuxserver;});
305315 obelisk = import (base + "/.obelisk/impl") {};
306316 };
307317 haskellPackageSets = {
lib/command/obelisk-command.cabal+2
@@ -27,6 +27,8 @@ library
2727 , io-streams
2828 , exceptions
2929 , logging-effect
30+ , lens
31+ , modern-uri
3032 , monad-loops
3133 , mtl
3234 , network
lib/command/src/Obelisk/Command.hs+11−1
@@ -174,6 +174,9 @@ deployInitOpts = DeployInitOpts
174174 <$> strArgument (action "directory" <> metavar "DEPLOYDIR" <> help "Path to a directory that it will create")
175175 <*> strOption (long "ssh-key" <> action "file" <> metavar "SSHKEY" <> help "Path to an ssh key that it will symlink to")
176176 <*> some (strOption (long "hostname" <> metavar "HOSTNAME" <> help "hostname of the deployment target"))
177+ <*> strOption (long "route" <> metavar "PUBLICROUTE" <> help "Publicly accessible URL of your app")
178+ <*> strOption (long "admin-email" <> metavar "ADMINEMAIL" <> help "Email address where administrative alerts will be sent")
179+ <*> flag True False (long "disable-https" <> help "Disable automatic https configuration for the backend")
177180 <*> strOption (long "upstream" <> value "origin" <> metavar "REMOTE" <> help "git remote to use for the src thunk" <> showDefault)
178181
179182type TeamID = String
@@ -194,6 +197,9 @@ data DeployInitOpts = DeployInitOpts
194197 { _deployInitOpts_outputDir :: FilePath
195198 , _deployInitOpts_sshKey :: FilePath
196199 , _deployInitOpts_hostname :: [String]
200+ , _deployInitOpts_route :: String
201+ , _deployInitOpts_adminEmail :: String
202+ , _deployInitOpts_enableHttps :: Bool
197203 , _deployInitOpts_remote :: String
198204 }
199205 deriving Show
@@ -373,7 +379,11 @@ ob = \case
373379 getThunkPtr' False root (T.pack $ _deployInitOpts_remote deployOpts)
374380 let sshKeyPath = _deployInitOpts_sshKey deployOpts
375381 hostname = _deployInitOpts_hostname deployOpts
376− deployInit thunkPtr (root </> "config") deployDir sshKeyPath hostname
382+ route = _deployInitOpts_route deployOpts
383+ adminEmail = _deployInitOpts_adminEmail deployOpts
384+ enableHttps = _deployInitOpts_enableHttps deployOpts
385+ deployInit thunkPtr (root </> "config") deployDir
386+ sshKeyPath hostname route adminEmail enableHttps
377387 DeployCommand_Push remoteBuilder -> deployPush "." $ case remoteBuilder of
378388 Nothing -> pure []
379389 Just RemoteBuilder_ObeliskVM -> (:[]) <$> VmBuilder.getNixBuildersArg
lib/command/src/Obelisk/Command/Deploy.hs+121−24
@@ -2,9 +2,12 @@
22{-# LANGUAGE LambdaCase #-}
33{-# LANGUAGE OverloadedStrings #-}
44{-# LANGUAGE ScopedTypeVariables #-}
5+{-# LANGUAGE ViewPatterns #-}
56module Obelisk.Command.Deploy where
67
8+import Control.Lens
79import Control.Monad
10+import Control.Monad.Catch (Exception (displayException), MonadThrow, throwM, try)
811import Control.Monad.IO.Class (liftIO)
912import Data.Bits
1013import Data.Default
@@ -18,6 +21,9 @@ import System.Environment (getEnvironment)
1821import System.FilePath
1922import System.Posix.Files
2023import System.Process (delegate_ctlc, env, proc)
24+import Text.URI (URI)
25+import qualified Text.URI as URI
26+import Text.URI.Lens
2127
2228import Obelisk.App (MonadObelisk)
2329import Obelisk.CliApp (Severity (..), callProcessAndLogOutput, failWith, putLog, withSpinner)
@@ -26,30 +32,48 @@ import Obelisk.Command.Project
2632import Obelisk.Command.Thunk
2733import Obelisk.Command.Utils
2834
29−deployInit :: MonadObelisk m => ThunkPtr -> FilePath -> FilePath -> FilePath -> [String] -> m ()
30−deployInit thunkPtr configDir deployDir sshKeyPath hostnames = do
31− hasConfigDir <- liftIO $ do
32− createDirectoryIfMissing True deployDir
33− doesDirectoryExist configDir
34− localKey <- liftIO (doesFileExist sshKeyPath) >>= \case
35− False -> failWith $ T.pack $ "ob deploy init: file does not exist: " <> sshKeyPath
36− True -> pure $ deployDir </> "ssh_key"
37− callProcessAndLogOutput (Notice, Error) $ proc "cp" [sshKeyPath, localKey]
38− liftIO $ setFileMode localKey $ ownerReadMode .|. ownerWriteMode
39− liftIO $ writeFile (deployDir </> "backend_hosts") $ unlines hostnames
35+deployInit
36+ :: MonadObelisk m
37+ => ThunkPtr
38+ -> FilePath
39+ -> FilePath
40+ -> FilePath
41+ -> [String] -- ^ hostnames
42+ -> String -- ^ route
43+ -> String -- ^ admin email
44+ -> Bool -- ^ enable https
45+ -> m ()
46+deployInit thunkPtr configDir deployDir sshKeyPath hostnames route adminEmail enableHttps = do
47+ (hasConfigDir, localKey) <- withSpinner ("Preparing " <> T.pack deployDir) $ do
48+ hasConfigDir <- liftIO $ do
49+ createDirectoryIfMissing True deployDir
50+ doesDirectoryExist configDir
51+ localKey <- liftIO (doesFileExist sshKeyPath) >>= \case
52+ False -> failWith $ T.pack $ "ob deploy init: file does not exist: " <> sshKeyPath
53+ True -> pure $ deployDir </> "ssh_key"
54+ callProcessAndLogOutput (Notice, Error) $
55+ proc "cp" [sshKeyPath, localKey]
56+ liftIO $ setFileMode localKey $ ownerReadMode .|. ownerWriteMode
57+ return $ (hasConfigDir, localKey)
58+ withSpinner "Validating configuration" $ do
59+ void $ getHostFromRoute enableHttps route -- make sure that hostname is present
4060 forM_ hostnames $ \hostname -> do
4161 putLog Notice $ "Verifying host keys (" <> T.pack hostname <> ")"
62+ -- Note: we can't use a spinner here as this function will prompt the user.
4263 verifyHostKey (deployDir </> "backend_known_hosts") localKey hostname
43− when hasConfigDir $ do
44− callProcessAndLogOutput (Notice, Error) $ proc "cp"
45− [ "-r"
46− , "-T"
47− , configDir
48− , deployDir </> "config"
49− ]
50− liftIO $ createThunk (deployDir </> "src") thunkPtr
51− liftIO $ setupObeliskImpl deployDir
52− initGit deployDir
64+ when hasConfigDir $ withSpinner "Importing project configuration" $ do
65+ callProcessAndLogOutput (Notice, Error) $
66+ proc "cp" [ "-r" , "-T" , configDir , deployDir </> "config"]
67+ withSpinner "Writing deployment configuration" $ do
68+ writeDeployConfig deployDir "backend_hosts" $ unlines hostnames
69+ writeDeployConfig deployDir "enable_https" $ show enableHttps
70+ writeDeployConfig deployDir "admin_email" adminEmail
71+ writeDeployConfig deployDir ("config" </> "common" </> "route") $ route
72+ withSpinner "Creating source thunk (./src)" $ liftIO $ do
73+ createThunk (deployDir </> "src") thunkPtr
74+ setupObeliskImpl deployDir
75+ withSpinner ("Initializing git repository (" <> T.pack deployDir <> ")") $
76+ initGit deployDir
5377
5478setupObeliskImpl :: FilePath -> IO ()
5579setupObeliskImpl deployDir = do
@@ -59,8 +83,11 @@ setupObeliskImpl deployDir = do
5983
6084deployPush :: MonadObelisk m => FilePath -> m [String] -> m ()
6185deployPush deployPath getNixBuilders = do
62− let backendHosts = deployPath </> "backend_hosts"
63− host <- liftIO $ fmap (T.unpack . T.strip) $ T.readFile backendHosts
86+ host <- readDeployConfig deployPath "backend_hosts"
87+ adminEmail <- readDeployConfig deployPath "admin_email"
88+ enableHttps <- read <$> readDeployConfig deployPath "enable_https"
89+ route <- readDeployConfig deployPath $ "config" </> "common" </> "route"
90+ routeHost <- getHostFromRoute enableHttps route
6491 let srcPath = deployPath </> "src"
6592 build = do
6693 builders <- getNixBuilders
@@ -70,7 +97,12 @@ deployPush deployPath getNixBuilders = do
7097 , _target_attr = Just "server"
7198 }
7299 , _nixBuildConfig_outLink = OutLink_None
73− , _nixBuildConfig_args = pure $ Arg "hostName" host
100+ , _nixBuildConfig_args =
101+ [ strArg "hostName" host
102+ , strArg "adminEmail" adminEmail
103+ , strArg "routeHost" routeHost
104+ , boolArg "enableHttps" enableHttps
105+ ]
74106 , _nixBuildConfig_builders = builders
75107 }
76108 return $ listToMaybe $ lines buildOutput
@@ -114,6 +146,7 @@ deployPush deployPath getNixBuilders = do
114146 ["-C", deployPath, "add", "."]
115147 callProcessAndLogOutput (Debug, Error) $ proc "git"
116148 ["-C", deployPath, "commit", "-m", "New deployment"]
149+ putLog Notice $ "Deployed => " <> T.pack route
117150 where
118151 callProcess' envMap cmd args = do
119152 processEnv <- Map.toList . (envMap <>) . Map.fromList <$> liftIO getEnvironment
@@ -131,6 +164,14 @@ deployMobile platform mobileArgs = withProjectRoot "." $ \root -> do
131164 result <- nixBuildAttrWithCache srcDir $ platform <> ".frontend"
132165 callProcessAndLogOutput (Notice, Error) $ proc (result </> "bin" </> "deploy") mobileArgs
133166
167+-- | Simplified deployment configuration mechanism. At one point we may revisit this.
168+writeDeployConfig :: MonadObelisk m => FilePath -> FilePath -> String -> m ()
169+writeDeployConfig deployDir fname = liftIO . writeFile (deployDir </> fname)
170+
171+readDeployConfig :: MonadObelisk m => FilePath -> FilePath -> m String
172+readDeployConfig deployDir fname = liftIO $ do
173+ fmap (T.unpack . T.strip) $ T.readFile $ deployDir </> fname
174+
134175verifyHostKey :: MonadObelisk m => FilePath -> FilePath -> String -> m ()
135176verifyHostKey knownHostsPath keyPath hostName =
136177 callProcessAndLogOutput (Notice, Warning) $ proc "ssh" $
@@ -145,3 +186,59 @@ sshArgs knownHostsPath keyPath askHostKeyCheck =
145186 , "-o", "StrictHostKeyChecking=" <> if askHostKeyCheck then "ask" else "yes"
146187 , "-i", keyPath
147188 ]
189+
190+-- common/route validation
191+-- TODO: move these to executable-config once the typed-config stuff is done.
192+
193+data InvalidRoute
194+ = InvalidRoute_NotHttps URI
195+ | InvalidRoute_MissingScheme URI
196+ | InvalidRoute_MissingHost URI
197+ | InvalidRoute_HasPort URI
198+ | InvalidRoute_HasPath URI
199+ deriving Show
200+
201+instance Exception InvalidRoute where
202+ displayException = \case
203+ InvalidRoute_MissingScheme uri -> route uri "must have an URI scheme"
204+ InvalidRoute_NotHttps uri -> route uri "must be HTTPS"
205+ InvalidRoute_MissingHost uri -> route uri "must contain a hostname"
206+ InvalidRoute_HasPort uri -> route uri "cannot specify port"
207+ InvalidRoute_HasPath uri -> route uri "cannot contain path"
208+ where
209+ route uri err = T.unpack $ "Route (" <> URI.render uri <> ") " <> err
210+
211+-- | Get the hostname from a https route
212+--
213+-- Fail if the route is invalid (i.e, no host present or scheme is not https)
214+getHostFromRoute
215+ :: MonadObelisk m
216+ => Bool -- ^ Ensure https?
217+ -> String
218+ -> m String
219+getHostFromRoute mustBeHttps route = do
220+ result :: Either InvalidRoute String <- try $ do
221+ validateCommonRouteAndGetHost mustBeHttps =<< URI.mkURI (T.strip $ T.pack route)
222+ either (failWith . T.pack . displayException) pure result
223+
224+validateCommonRouteAndGetHost
225+ :: (MonadThrow m, MonadObelisk m)
226+ => Bool -- ^ Ensure https?
227+ -> URI
228+ -> m String
229+validateCommonRouteAndGetHost mustBeHttps uri = do
230+ case uri ^? uriScheme of
231+ Just (Just (URI.unRText -> s)) -> case (mustBeHttps, s) of
232+ (False, _) -> pure ()
233+ (True, "https") -> pure ()
234+ _ -> throwM $ InvalidRoute_NotHttps uri
235+ _ -> throwM $ InvalidRoute_MissingScheme uri
236+ case uri ^. uriPath of
237+ [] -> pure ()
238+ _path -> throwM $ InvalidRoute_HasPath uri
239+ case uri ^? uriAuthority . _Right . authPort of
240+ Just (Just _port) -> throwM $ InvalidRoute_HasPort uri
241+ _ -> pure ()
242+ case uri ^? uriAuthority . _Right . authHost of
243+ Nothing -> throwM $ InvalidRoute_MissingHost uri
244+ Just sslHost -> return $ T.unpack $ URI.unRText sslHost
lib/command/src/Obelisk/Command/Nix.hs+20−5
@@ -1,4 +1,5 @@
11{-# LANGUAGE FlexibleContexts #-}
2+{-# LANGUAGE LambdaCase #-}
23{-# LANGUAGE MultiParamTypeClasses #-}
34{-# LANGUAGE OverloadedStrings #-}
45{-# LANGUAGE ScopedTypeVariables #-}
@@ -8,8 +9,11 @@ module Obelisk.Command.Nix
89 , Target (..)
910 , OutLink (..)
1011 , Arg (..)
12+ , boolArg
13+ , strArg
1114 ) where
1215
16+import Data.Bool (bool)
1317import Data.Default
1418import Data.List (intercalate)
1519import Data.Monoid ((<>))
@@ -40,10 +44,21 @@ instance Default Target where
4044 , _target_attr = Nothing
4145 }
4246
43−data Arg = Arg
44− { _arg_key :: String
45− , _arg_value :: String
46− }
47+data Arg
48+ = Arg_Str String String
49+ | Arg_Expr String String
50+ deriving (Eq, Show)
51+
52+strArg :: String -> String -> Arg
53+strArg k = Arg_Str k
54+
55+boolArg :: String -> Bool -> Arg
56+boolArg k = Arg_Expr k . bool "false" "true"
57+
58+cliFromArgs :: [Arg] -> [String]
59+cliFromArgs = concatMap $ \case
60+ Arg_Str k v -> ["--argstr", k, v]
61+ Arg_Expr k v -> ["--arg", k, v]
4762
4863data NixBuildConfig = NixBuildConfig
4964 { _nixBuildConfig_target :: Target
@@ -65,7 +80,7 @@ nixBuild cfg = withSpinner' ("Running nix-build on " <> desc) (Just $ const $ "B
6580 attrArg = case attr of
6681 Nothing -> []
6782 Just a -> ["-A", a]
68− args = mconcat [["--argstr", k, v] | Arg k v <- _nixBuildConfig_args cfg]
83+ args = cliFromArgs $ _nixBuildConfig_args cfg
6984 outLink = case _nixBuildConfig_outLink cfg of
7085 OutLink_Default -> []
7186 OutLink_None -> ["--no-out-link"]
lib/https/default.nixdeleted−7
@@ -1,7 +0,0 @@
1−# DO NOT HAND-EDIT THIS FILE
2−import ((import <nixpkgs> {}).fetchFromGitHub (
3− let json = builtins.fromJSON (builtins.readFile ./github.json);
4− in { inherit (json) owner repo rev sha256;
5− private = json.private or false;
6− }
7−))
lib/https/github.jsondeleted−6
@@ -1,6 +0,0 @@
1−{
2− "owner": "obsidiansystems",
3− "repo": "obelisk-https",
4− "rev": "5bc5cd81b669a0ea1b4415a1be8fd8c5a0680f0b",
5− "sha256": "105n7pxykz02n8ilgjznssmxa8hszclg21nqiyqdr64bcrxjyxf4"
6−}
migration/07900e6c41ab9a78917f66fcf7e0d9b8-8c84f81d9b09a5e0f028f3c6341ffb36/obelisk-handoffnew file

No changes to the content.

migration/07900e6c41ab9a78917f66fcf7e0d9b8-8c84f81d9b09a5e0f028f3c6341ffb36/obelisk-upgradenew file+15
@@ -0,0 +1,15 @@
1+# ob deploy: HTTPs support
2+
3+Obelisk deployments now support HTTPs. You must upgrade the deployment repo
4+associated with the obelisk project as follows:
5+
6+```
7+cd /your/project's/deployment/repo
8+echo True > enable_https
9+echo admin@yoursite.com > admin_email
10+echo https://yoursite.com/ > config/common/route
11+git add ... && git commit ...
12+ob deploy push
13+```
14+
15+If you do not wish to enable HTTPs, write `False` to `enable_https` instead.