git.iko.soy/typeable/octopod
on master help sign in
Added more detailed errors (#31)

* Added more failure statuses * Added documentation

commit6c6a4dce2c7b9dc61c2efabb95525ed89375c245 authoriko <ilyakooo0@gmail.com> dateFri 5 Feb 2021, 13:09 (6 years ago) parentf0d23b0
12 files changed+115−17
docs/en/Control_scripts.md+5 docs/ru/Control_scripts.md+5 migrations/deploy/add_detailed_failures.sql+14 migrations/revert/add_detailed_failures.sql+9 migrations/sqitch.plan+1 migrations/verify/add_detailed_failures.sql+10 octopod-backend/octopod-backend.cabal+2 octopod-backend/src/Database/PostgreSQL/Simple/Instances.hs+40 octopod-backend/src/Octopod/Server.hs+19−14 octopod-common/src/Common/Types.hs+8−1 octopod-common/src/Common/Utils.hs+1−1 octopod-frontend/src/Frontend/Utils.hs+1−1
docs/en/Control_scripts.md+5
@@ -178,6 +178,11 @@ This script checks the status of the deployment.
178178
179179If the script exits with `0`, it means that the deployment is healthy and up. If the script exits with a non-zero exit code, it means that the deployment is not healthy or down.
180180
181+You can specify exactly what error occured using exit codes:
182+- `1` – a generic failure.
183+- `2` – the deployment is partially down (some containers are unhealthy).
184+- `3` – tag mismatch, the deployment has not been updated to the expected version.
185+
181186This script receives only [the default command-line arguments](#general-behavior) as input.
182187
183188#### Execution example
docs/ru/Control_scripts.md+5
@@ -111,6 +111,11 @@ echo "{\"Deployments\": [{\"ResourceName\": \"app-${name}\", \"Namespace\": \"${
111111В примере выше используется [_Kubedog_][kubedog].
112112Предполагается, что пользователь сам установит [_Kubedog_][kubedog]: сохранит его в `/` контейнера с _Control scripts_ или установит его в `$HOME` из [init](#init).
113113
114+При помощи exit code можно также сигнализировать какая именно ошибка произошла:
115+- `1` – Стандартная ошибка.
116+- `2` – Частичная недоступность (некоторые из подов недоступны).
117+- `3` – Несоответствие тегов (тег некорректно обновился).
118+
114119## cleanup
115120
116121Реализация очистки ресурсов развертывания.
migrations/deploy/add_detailed_failures.sqlnew file+14
@@ -0,0 +1,14 @@
1+-- Deploy octopod:add_detailed_failures to pg
2+ BEGIN;
3+
4+
5+ALTER TYPE statuses RENAME VALUE 'Failure' TO 'GenericFailure';
6+
7+
8+ALTER TYPE statuses ADD VALUE IF NOT EXISTS 'TagMismatch';
9+
10+
11+ALTER TYPE statuses ADD VALUE IF NOT EXISTS 'PartialAvailability';
12+
13+
14+COMMIT;
migrations/revert/add_detailed_failures.sqlnew file+9
@@ -0,0 +1,9 @@
1+-- Revert octopod:add_detailed_failures from pg
2+ BEGIN;
3+
4+
5+ALTER TYPE statuses RENAME VALUE 'GenericFailure' TO 'Failure';
6+
7+-- NOTE: not a full revert. Removing enum values is a pain.
8+
9+COMMIT;
migrations/sqitch.plan+1
@@ -13,3 +13,4 @@ create_deployment_metadata 2020-08-14T12:11:42Z Typeable LLC <octopod@typeable.i
1313rename_elements_of_scope_enum 2020-08-19T09:38:40Z Typeable LLC <octopod@typeable.io> # Rename elements of 'scope' enum
1414rename-delete-to-archive 2020-11-23T10:53:10Z Ilya <octopod@typeable.io> # Renamed delete to archive
1515rename_delete_to_archive_2 2020-11-26T08:28:58Z Typeable LLC <octopod@typeable.io> # Renamed delete to archive
16+add_detailed_failures 2021-02-04T11:01:15Z Typeable LLC <octopod@typeable.io> # Added more failure states
migrations/verify/add_detailed_failures.sqlnew file+10
@@ -0,0 +1,10 @@
1+-- Verify octopod:add_detailed_failures on pg
2+ BEGIN;
3+
4+
5+SELECT ('GenericFailure'::statuses,
6+ 'TagMismatch'::statuses,
7+ 'PartialAvailability'::statuses);
8+
9+
10+ROLLBACK;
octopod-backend/octopod-backend.cabal+2
@@ -44,6 +44,7 @@ library
4444 Orphans
4545 TLS
4646 Types
47+ Database.PostgreSQL.Simple.Instances
4748 hs-source-dirs:
4849 src
4950 build-depends:
@@ -91,6 +92,7 @@ library
9192 , x509
9293 , x509-store
9394 , x509-validation
95+ , containers
9496 default-language: Haskell2010
9597
9698executable octopod-exe
octopod-backend/src/Database/PostgreSQL/Simple/Instances.hsnew file+40
@@ -0,0 +1,40 @@
1+{-# OPTIONS_GHC -Wno-orphans #-}
2+
3+module Database.PostgreSQL.Simple.Instances
4+ (
5+ ) where
6+
7+import Common.Types
8+import Control.Applicative
9+import Control.Arrow
10+import qualified Data.Map as M
11+import Data.Text (Text)
12+import Database.PostgreSQL.Simple.FromField
13+import Database.PostgreSQL.Simple.ToField
14+
15+deploymentStatusText :: DeploymentStatus -> Text
16+deploymentStatusText Running = "Running"
17+deploymentStatusText (Failure GenericFailure) = "GenericFailure"
18+deploymentStatusText (Failure TagMismatch) = "TagMismatch"
19+deploymentStatusText (Failure PartialAvailability) = "PartialAvailability"
20+deploymentStatusText CreatePending = "CreatePending"
21+deploymentStatusText UpdatePending = "UpdatePending"
22+deploymentStatusText ArchivePending = "ArchivePending"
23+deploymentStatusText Archived = "Archived"
24+
25+instance ToField DeploymentStatus where
26+ toField = toField @Text . deploymentStatusText
27+
28+instance FromField DeploymentStatus where
29+ fromField f b = fromField f b >>= maybe empty return . flip M.lookup m
30+ where
31+ m = M.fromList . fmap (deploymentStatusText &&& id) $
32+ [ Running
33+ , Failure GenericFailure
34+ , Failure TagMismatch
35+ , Failure PartialAvailability
36+ , CreatePending
37+ , UpdatePending
38+ , ArchivePending
39+ , Archived
40+ ]
octopod-backend/src/Octopod/Server.hs+19−14
@@ -26,6 +26,7 @@ import Data.Text (lines, pack, unpack, unwords)
2626import Data.Text.IO (hGetContents)
2727import Data.Traversable
2828import Database.PostgreSQL.Simple
29+import Database.PostgreSQL.Simple.Instances ()
2930import Network.Wai.Handler.Warp
3031import Network.Wai.Handler.WarpTLS
3132import Options.Generic
@@ -309,7 +310,7 @@ getFullInfo listType = do
309310 dMeta <- selectDeploymentMetadata conn n
310311 pure $ do
311312 let dep = (Deployment n t appOvs depOvs)
312− DeploymentFullInfo dep (read st) a dMeta ct ut
313+ DeploymentFullInfo dep st a dMeta ct ut
313314 liftIO . logInfo l $ "get deployments: " <> (pack . show $ deployments)
314315 return deployments
315316 where
@@ -344,7 +345,7 @@ createH dep = do
344345 createDep :: PgPool -> Deployment -> IO [Only Int]
345346 createDep p Deployment { name = n, tag = t } =
346347 withResource p $ \conn ->
347− query conn q (n, t, show CreatePending)
348+ query conn q (n, t, CreatePending)
348349 failIfImageNotFound (name dep) (tag dep)
349350 failIfGracefulShutdownActivated
350351 res :: Either SqlError [Only Int] <- liftIO . try $ createDep pgPool dep
@@ -554,7 +555,7 @@ archiveDeployment p dName = withResource p $ \conn -> do
554555 \SET archived = 't', archived_at = now(), \
555556 \status = ?, status_updated_at = now() \
556557 \WHERE name = ?"
557− execute conn q (show ArchivePending, dName)
558+ execute conn q (ArchivePending, dName)
558559
559560-- | Handles the 'update' request.
560561updateH :: DeploymentName -> DeploymentUpdate -> AppM CommandResponse
@@ -743,7 +744,7 @@ updateDeployment conn dName dTag = do
743744 \SET tag = ?, updated_at = now(), \
744745 \status = ?, status_updated_at = now() \
745746 \WHERE name = ?"
746− execute conn q (dTag, show UpdatePending, dName)
747+ execute conn q (dTag, UpdatePending, dName)
747748
748749-- | Handles the 'info' request of the Web UI API.
749750infoH :: DeploymentName -> AppM [DeploymentInfo]
@@ -1069,7 +1070,6 @@ upsertDeploymentMetadata
10691070 -> [DeploymentMetadata]
10701071 -> IO ()
10711072upsertDeploymentMetadata pgPool dName dMetadatas = do
1072− let
10731073 withResource pgPool $ \conn -> withTransactionSerializable conn $ do
10741074 void $ execute
10751075 conn
@@ -1148,11 +1148,11 @@ runStatusUpdater state = do
11481148 "UPDATE deployments SET checked_at = now() WHERE name = ? and status = ?"
11491149
11501150 forever $ do
1151− rows :: [(DeploymentName, Text, Int)] <- liftIO $
1151+ rows :: [(DeploymentName, DeploymentStatus, Int)] <- liftIO $
11521152 withResource pgPool $ \conn -> query conn selectDeps (Only interval)
11531153 let
11541154 checkList :: [(DeploymentName, DeploymentStatus, Timestamp)] =
1155− (\(n, s, t) -> (n, read . unpack $ s, coerce t)) <$> rows
1155+ (\(n, s, t) -> (n, s, coerce t)) <$> rows
11561156 checkResult <- for checkList $ \(dName, dStatus, ts) -> do
11571157 let
11581158 args =
@@ -1169,8 +1169,8 @@ runStatusUpdater state = do
11691169 for (zip checkList checkResult) $ \((dName, oldSt, _), (_, newSt, _)) ->
11701170 withResource pgPool $ \conn ->
11711171 if oldSt == newSt
1172− then execute conn updateCheckedAt (dName, show oldSt)
1173− else execute conn updateStatus (show newSt, dName, show oldSt)
1172+ then execute conn updateCheckedAt (dName, oldSt)
1173+ else execute conn updateStatus (newSt, dName, oldSt)
11741174 if checkList == checkResult
11751175 then pure ()
11761176 else sendReloadEvent state
@@ -1185,14 +1185,19 @@ newStatus
11851185 -> DeploymentStatus
11861186newStatus ExitSuccess ArchivePending _ _ = Archived
11871187newStatus ExitSuccess _ _ _ = Running
1188−newStatus (ExitFailure _) Running _ _ = Failure
1189−newStatus (ExitFailure _) CreatePending ts timeout | ts > coerce timeout =
1190− Failure
1191−newStatus (ExitFailure _) UpdatePending ts timeout | ts > coerce timeout =
1192− Failure
1188+newStatus (ExitFailure n) Running _ _ = Failure $ failureStatusType n
1189+newStatus (ExitFailure n) CreatePending ts timeout | ts > coerce timeout =
1190+ Failure $ failureStatusType n
1191+newStatus (ExitFailure n) UpdatePending ts timeout | ts > coerce timeout =
1192+ Failure $ failureStatusType n
11931193newStatus (ExitFailure _) ArchivePending _ _ = ArchivePending
11941194newStatus (ExitFailure _) oldStatus _ _ = oldStatus
11951195
1196+failureStatusType :: Int -> FailureType
1197+failureStatusType 2 = PartialAvailability
1198+failureStatusType 3 = TagMismatch
1199+failureStatusType _ = GenericFailure
1200+
11961201-- | Checks if graceful shutdown has been activated activated.
11971202-- Returns 405 'Graceful shutdown activated' response
11981203-- if graceful shutdown has been activated.
octopod-common/src/Common/Types.hs+8−1
@@ -86,7 +86,7 @@ newtype ProjectName = ProjectName { uProjectName :: Text }
8686
8787data DeploymentStatus
8888 = Running
89− | Failure
89+ | Failure FailureType
9090 | CreatePending
9191 | UpdatePending
9292 | ArchivePending
@@ -94,6 +94,13 @@ data DeploymentStatus
9494 deriving (Generic, Read, Show, Eq)
9595 deriving (FromJSON, ToJSON) via Snake DeploymentStatus
9696
97+data FailureType
98+ = GenericFailure
99+ | TagMismatch
100+ | PartialAvailability
101+ deriving (Generic, Read, Show, Eq)
102+ deriving (FromJSON, ToJSON) via Snake FailureType
103+
97104data Deployment = Deployment
98105 { name :: DeploymentName
99106 , tag :: DeploymentTag
octopod-common/src/Common/Utils.hs+1−1
@@ -34,7 +34,7 @@ dfiName = field @"deployment" . field @"name"
3434isPending :: DeploymentStatus -> Bool
3535isPending = \case
3636 Running -> False
37− Failure -> False
37+ Failure _ -> False
3838 Archived -> False
3939 CreatePending -> True
4040 UpdatePending -> True
octopod-frontend/src/Frontend/Utils.hs+1−1
@@ -358,7 +358,7 @@ statusWidget stDyn = do
358358 pendingWidget = divClass "loading loading--status-alike"
359359 dyn_ $ stDyn' <&> \case
360360 Running -> divClass "status status--success" $ text "Running"
361− Failure -> divClass "status status--failure" $ text "Failure"
361+ Failure _ -> divClass "status status--failure" $ text "Failure"
362362 CreatePending -> pendingWidget $ text "Creating..."
363363 UpdatePending -> pendingWidget $ text "Updating..."
364364 ArchivePending -> pendingWidget $ text "Archiving..."