Support only the bounded cache, with default HASURA_GRAPHQL_QUERY_PLAN_CACHE_SIZE of 4000. Closes #5363

CHANGELOG.md

@@ -7,6 +7,7 @@
 (Add entries here in the order of: server, console, cli, docs, others)
 
 - server: bugfix to allow HASURA_GRAPHQL_QUERY_PLAN_CACHE_SIZE of 0 (#5363)
+- server: support only a bounded plan cache, with a default size of 4000 (closes #5363)
 - console: update sidebar icons for different action and trigger types
 
 ## `v1.3.0`

server/graphql-engine.cabal

@@ -244,10 +244,7 @@ library
                      , Data.Time.Clock.Units
                      , Data.URL.Template
                      , Hasura.App
-                     , Hasura.Cache
                      , Hasura.Cache.Bounded
-                     , Hasura.Cache.Types
-                     , Hasura.Cache.Unbounded
                      , Hasura.Db
                      , Hasura.EncJSON
                      , Hasura.Eventing.Common

server/src-bench-cache/Main.hs

@@ -15,13 +15,10 @@ import           Data.Traversable
 import qualified Data.Vector                   as V
 import           Data.Word
 import           GHC.Clock
-import qualified Hasura.Cache.Bounded          as B
-import qualified Hasura.Cache.Unbounded        as U
+import qualified Hasura.Cache.Bounded          as Cache
 import           Prelude
 import           System.Random.MWC             as Rand
 import           System.Random.MWC.Probability as Rand
--- higher level interface to above, combined:
-import qualified Hasura.Cache                  as Cache
 
 -- Benchmarks for code backing the plan cache.
 
@@ -42,15 +39,12 @@ main = defaultMain [
       bgroup "insert x1000" [
         -- use perRunEnv so we can be sure we're not triggering cache
         -- evictions in bounded due to long bootstrap batch runs
-        bench "unbounded" $
-          perRunEnv (U.initialise) $ \cache ->
-            V.mapM_ (\k -> Cache.insert k k cache) rs
-      , bench "bounded" $
-          perRunEnv (B.initialise 4000) $ \cache ->
+        bench "bounded" $
+          perRunEnv (Cache.initialise 4000) $ \cache ->
             V.mapM_ (\k -> Cache.insert k k cache) rs
       -- an eviction on each insert, all LRU counters at zero. Simulates a scan.
       , bench "bounded evicting scan" $
-          let preloaded = populate 5000 (B.initialise 5000) B.insertAllStripes
+          let preloaded = populate 5000 (Cache.initialise 5000) Cache.insertAllStripes
            in perRunEnv (preloaded) $ \(cache, _) ->
                 V.mapM_ (\k -> Cache.insert k k cache) rs
       ]
@@ -113,14 +107,11 @@ realisticBenches name wrk =
         -- For oversubscribed case: can we see descheduled threads blocking global progress?
         flip map [2,100] $ \threadsPerHEC ->
           bgroup (show threadsPerHEC <>"xCPUs threads") [
-            bench "unbounded" $
-              perRunEnv (Cache.initialise Nothing) $ \cache ->
-                go threadsPerHEC cache payloads
-          , bench "bounded effectively unbounded" $
-              perRunEnv (Cache.initialise $ Just 40000) $ \cache ->
+            bench "bounded effectively unbounded" $
+              perRunEnv (Cache.initialise 40000) $ \cache ->
                 go threadsPerHEC cache payloads
           , bench "bounded 10pct ideal capacity" $
-              perRunEnv (Cache.initialise $ Just 2700) $ \cache ->
+              perRunEnv (Cache.initialise 2700) $ \cache ->
                 go threadsPerHEC cache payloads
           ]
     --  660K uniques, 40% in top 10% , 30% in top 1%, 33% cache hits ideally
@@ -128,20 +119,17 @@ realisticBenches name wrk =
       bgroup "realistic distribution" $
         flip map [2,100] $ \threadsPerHEC ->
           bgroup (show threadsPerHEC <>"xCPUs threads") [
-            bench "unbounded" $
-              perRunEnv (Cache.initialise Nothing) $ \cache ->
-                go threadsPerHEC cache payloads
-          , bench "bounded maxBound (10pct ideal capacity)" $
+            bench "bounded maxBound (10pct ideal capacity)" $
               -- this is our largest possible cache size will necessarily evict
-              perRunEnv (Cache.initialise $ Just maxBound) $ \cache ->
+              perRunEnv (Cache.initialise maxBound) $ \cache ->
                 go threadsPerHEC cache payloads
           , bench "bounded 6000 (1pct ideal capacity)" $
-              perRunEnv (Cache.initialise $ Just 6000) $ \cache ->
+              perRunEnv (Cache.initialise 6000) $ \cache ->
                 go threadsPerHEC cache payloads
           ]
   ]
   where
-    go :: Int -> Cache.Cache Int Int -> [Int] -> IO ()
+    go :: Int -> Cache.BoundedCache Int Int -> [Int] -> IO ()
     go threadFactor cache payload = do
       bef <- getMonotonicTimeNSec
       -- So that `go 0 ...`  will give us a single thread:
@@ -207,14 +195,7 @@ burnCycles = go 0XBEEF where
 readBenches :: Int -> Benchmark
 readBenches n =
   bgroup ("size "<>show n) [
-    env (populate n U.initialise U.insertAllStripes) $ \ ~(cache, k)->
-      bgroup "unbounded" [
-        bench "hit" $
-          nfAppIO (\k' -> Cache.lookup k' cache) k
-      , bench "miss" $
-          nfAppIO (\k' -> Cache.lookup k' cache) 0xDEAD
-      ]
-  , env (populate n (B.initialise (fromIntegral $ n*2)) B.insertAllStripes) $ \ ~(cache, k)->
+    env (populate n (Cache.initialise (fromIntegral $ n*2)) Cache.insertAllStripes) $ \ ~(cache, k)->
       bgroup "bounded" [
         bench "hit" $
           nfAppIO (\k' -> Cache.lookup k' cache) k
@@ -273,9 +254,5 @@ zipfianRandomInts n sk = do
 
 
 -- noops, orphans:
-instance NFData (B.BoundedCache k v) where
-  rnf _ = ()
-instance NFData (U.UnboundedCache k v) where
-  rnf _ = ()
-instance NFData (Cache.Cache k v) where
+instance NFData (Cache.BoundedCache k v) where
   rnf _ = ()

server/src-lib/Hasura/Cache.hs

@@ -1,21 +0,0 @@
-module Hasura.Cache
-  ( module Hasura.Cache.Types
-  , B.CacheSize(..)
-  , B.parseCacheSize
-  , initialise
-  ) where
-
-import           Hasura.Prelude         hiding (lookup)
-
-import           Control.Concurrent (getNumCapabilities)
-
-import           Hasura.Cache.Types
-import qualified Hasura.Cache.Bounded   as B
-import qualified Hasura.Cache.Unbounded as U
-
-initialise :: (Hashable k, Ord k) => Maybe B.CacheSize -> IO (Cache k v)
-initialise cacheSizeM = do
-  stripes <- getNumCapabilities
-  case cacheSizeM of
-    Nothing        -> Cache <$> U.initialise
-    Just cacheSize -> Cache <$> B.initialise stripes cacheSize

server/src-lib/Hasura/Cache/Bounded.hs

@@ -10,18 +10,21 @@ module Hasura.Cache.Bounded
   , parseCacheSize
 
   , initialise
+  , initialiseStripes
   , insertAllStripes
+  , lookup
+  , insert
+  , clear
+  , getEntries
 
   -- * Exposed for testing
   , checkInvariants
   , getEntriesRecency
-  , CacheObj(..)
   ) where
 
 import           Hasura.Prelude     hiding (lookup)
-import           Hasura.Cache.Types
 
-import           Control.Concurrent (myThreadId, threadCapability)
+import           Control.Concurrent (getNumCapabilities, myThreadId, threadCapability)
 import           Data.Word          (Word16)
 
 import qualified Data.Aeson         as J
@@ -146,25 +149,39 @@ insertLocal (LocalCacheRef ref) k v =
 -- accessed in parallel.
 newtype BoundedCache k v = BoundedCache (V.Vector (LocalCacheRef k v))
 
-instance (Hashable k, Ord k) => CacheObj (BoundedCache k v) k v where
-  clear (BoundedCache caches) =
-    V.mapM_ clearLocal caches
-  insert k v striped = do
-    localHandle <- getLocal striped
-    insertLocal localHandle k v
-  lookup k striped = do
-    localHandle <- getLocal striped
-    lookupLocal localHandle k
-  getEntries (BoundedCache localCaches) =
-    mapM getLocalEntries $ V.toList localCaches
+
+
+lookup :: (Hashable k, Ord k) => k -> BoundedCache k v -> IO (Maybe v)
+lookup k striped = do
+  localHandle <- getLocal striped
+  lookupLocal localHandle k
+
+insert :: (Hashable k, Ord k) => k -> v -> BoundedCache k v -> IO ()
+insert k v striped = do
+  localHandle <- getLocal striped
+  insertLocal localHandle k v
+
+clear :: BoundedCache k v -> IO ()
+clear (BoundedCache caches) =
+  V.mapM_ clearLocal caches
+
+getEntries :: (Hashable k, Ord k)=> BoundedCache k v -> IO [[(k, v)]]
+getEntries (BoundedCache localCaches) =
+  mapM getLocalEntries $ V.toList localCaches
+
+-- | Creates a new BoundedCache of the specified size, with one stripe per capability.
+initialise :: CacheSize -> IO (BoundedCache k v)
+initialise sz = do 
+  caps <- getNumCapabilities
+  initialiseStripes caps sz
 
 -- | Creates a new BoundedCache of the specified size, for each stripe
-initialise 
+initialiseStripes
   :: Int 
   -- ^ Stripes; to minimize contention this should probably match the number of capabilities.
   -> CacheSize 
   -> IO (BoundedCache k v)
-initialise stripes capacity = do
+initialiseStripes stripes capacity = do
   BoundedCache <$> V.replicateM stripes (initLocalCache capacity)
 
 

server/src-lib/Hasura/Cache/Types.hs

@@ -1,22 +0,0 @@
-{-# LANGUAGE ExistentialQuantification #-}
-
-module Hasura.Cache.Types
-  ( CacheObj(..)
-  , Cache(..)
-  ) where
-
-import           Hasura.Prelude         hiding (lookup)
-
-class (Hashable k, Ord k) => CacheObj c k v | c -> k v where
-  lookup :: k -> c -> IO (Maybe v)
-  insert :: k -> v -> c -> IO ()
-  clear :: c -> IO ()
-  getEntries :: c -> IO [[(k, v)]]
-
-data Cache k v = forall c . CacheObj c k v => Cache c
-
-instance (Hashable k, Ord k) => CacheObj (Cache k v) k v where
-  lookup k (Cache c) = lookup k c
-  insert k v (Cache c) = insert k v c
-  clear (Cache c) = clear c
-  getEntries (Cache c) = getEntries c

server/src-lib/Hasura/Cache/Unbounded.hs

@@ -1,83 +0,0 @@
-{-| An in-memory, unbounded, capability-local cache implementation. By making the cache
-capability-local, data may be recomputed up to once per capability (which usually means up to once
-per OS thread), but write contention from multiple threads is unlikely. -}
-module Hasura.Cache.Unbounded
-  ( UnboundedCache
-  , initialise
-  , insertAllStripes
-  ) where
-
-import           Hasura.Prelude      hiding (lookup)
-import           Hasura.Cache.Types
-
-import           Control.Concurrent  (getNumCapabilities, myThreadId, threadCapability)
-
-import qualified Data.HashMap.Strict as Map
-import qualified Data.IORef          as IORef
-import qualified Data.Vector         as V
-
-newtype LocalCacheRef k v = LocalCacheRef (IORef.IORef (Map.HashMap k v))
-
-getEntriesLocal
-  :: LocalCacheRef k v -> IO [(k, v)]
-getEntriesLocal (LocalCacheRef ioRef) =
-  Map.toList <$> IORef.readIORef ioRef
-
--- | Create a new LC cache of the given size.
-initialiseLocal :: IO (LocalCacheRef k v)
-initialiseLocal = LocalCacheRef <$> IORef.newIORef Map.empty
-
-clearLocal :: LocalCacheRef k v -> IO ()
-clearLocal (LocalCacheRef ref)=
-  IORef.atomicModifyIORef' ref $ const (Map.empty, ())
-
-lookupLocal :: (Hashable k, Eq k) => LocalCacheRef k v -> k -> IO (Maybe v)
-lookupLocal (LocalCacheRef ref) k =
-  Map.lookup k <$> IORef.readIORef ref
-
-insertLocal :: (Hashable k, Eq k) => LocalCacheRef k v -> k -> v -> IO ()
-insertLocal (LocalCacheRef ref) k v =
-  IORef.atomicModifyIORef' ref $ \c -> (Map.insert k v c, ())
-
--- | Using a stripe of multiple handles can improve the performance in
--- the case of concurrent accesses since several handles can be
--- accessed in parallel.
-newtype UnboundedCache k v = UnboundedCache (V.Vector (LocalCacheRef k v))
-
-instance (Hashable k, Ord k) => CacheObj (UnboundedCache k v) k v where
-  lookup k striped = do
-    localHandle <- getLocal striped
-    lookupLocal localHandle k
-  insert k v striped = do
-    localHandle <- getLocal striped
-    insertLocal localHandle k v
-  clear (UnboundedCache caches) =
-    V.mapM_ clearLocal caches
-  getEntries (UnboundedCache localCaches) =
-    mapM getEntriesLocal $ V.toList localCaches
-
--- | Create a new 'StripedHandle' with the given number of stripes and
--- the given capacity for each stripe.
-initialise :: IO (UnboundedCache k v)
-initialise = do
-  capabilities <- getNumCapabilities
-  UnboundedCache <$> V.replicateM capabilities initialiseLocal
-
-{-# INLINE getLocal #-}
-getLocal :: UnboundedCache k v -> IO (LocalCacheRef k v)
-getLocal (UnboundedCache handles) = do
-
-  (i, _) <- myThreadId >>= threadCapability
-
-  -- The number of capabilities can grow dynamically so make sure we wrap
-  -- around when indexing.
-  let j = i `mod` V.length handles
-
-  return $ handles V.! j
-
--- | Insert into all stripes (non-atomically).
-insertAllStripes
-  :: (Hashable k, Eq k) =>  k -> v -> UnboundedCache k v ->IO ()
-insertAllStripes k v (UnboundedCache handles) = do
-  forM_ handles $ \localHandle->
-    insertLocal localHandle k v

server/src-lib/Hasura/GraphQL/Execute/Plan.hs

@@ -17,7 +17,7 @@ import           Hasura.Prelude
 import           Hasura.RQL.Types
 import           Hasura.Session
 
-import qualified Hasura.Cache                           as Cache
+import qualified Hasura.Cache.Bounded                   as Cache
 import qualified Hasura.GraphQL.Execute.LiveQuery       as LQ
 import qualified Hasura.GraphQL.Execute.Query           as EQ
 import qualified Hasura.GraphQL.Resolve                 as R
@@ -45,7 +45,7 @@ instance J.ToJSON PlanId where
     ]
 
 newtype PlanCache
-  = PlanCache {_unPlanCache :: Cache.Cache PlanId ReusablePlan}
+  = PlanCache {_unPlanCache :: Cache.BoundedCache PlanId ReusablePlan}
 
 data ReusablePlan
   = RPQuery !EQ.ReusableQueryPlan ![R.QueryRootFldUnresolved]
@@ -57,7 +57,7 @@ instance J.ToJSON ReusablePlan where
     RPSubs subsPlan     -> J.toJSON subsPlan
 
 newtype PlanCacheOptions
-  = PlanCacheOptions { unPlanCacheSize :: Maybe Cache.CacheSize }
+  = PlanCacheOptions { unPlanCacheSize :: Cache.CacheSize }
   deriving (Show, Eq)
 $(J.deriveJSON (J.aesonDrop 2 J.snakeCase) ''PlanCacheOptions)
 

server/src-lib/Hasura/Server/Init.hs

@@ -22,7 +22,7 @@ import           Data.Time                        (NominalDiffTime)
 import           Network.Wai.Handler.Warp         (HostPreference)
 import           Options.Applicative
 
-import qualified Hasura.Cache                     as Cache
+import qualified Hasura.Cache.Bounded             as Cache
 import qualified Hasura.GraphQL.Execute           as E
 import qualified Hasura.GraphQL.Execute.LiveQuery as LQ
 import qualified Hasura.Logging                   as L
@@ -156,7 +156,8 @@ mkServeOptions rso = do
   enabledLogs <- maybe L.defaultEnabledLogTypes (Set.fromList) <$>
                  withEnv (rsoEnabledLogTypes rso) (fst enabledLogsEnv)
   serverLogLevel <- fromMaybe L.LevelInfo <$> withEnv (rsoLogLevel rso) (fst logLevelEnv)
-  planCacheOptions <- E.PlanCacheOptions <$> withEnv (rsoPlanCacheSize rso) (fst planCacheSizeEnv)
+  planCacheOptions <- E.PlanCacheOptions . fromMaybe 4000 <$>
+                      withEnv (rsoPlanCacheSize rso) (fst planCacheSizeEnv)
   devMode <- withEnvBool (rsoDevMode rso) $ fst devModeEnv
   adminInternalErrors <- fromMaybe True <$> -- Default to `true` to enable backwards compatibility
                                 withEnv (rsoAdminInternalErrors rso) (fst adminInternalErrorsEnv)
@@ -841,12 +842,19 @@ enableAllowlistEnv =
   , "Only accept allowed GraphQL queries"
   )
 
+-- NOTES re. default:
+--     There's a lot of guesswork and estimation here. Based on our test suite
+--   the average in-memory payload for a cache entry is 7kb, with the largest
+--   being 70kb. 128mb per-HEC seems like a reasonable default upper bound
+--   (note there is a distinct stripe per-HEC, for now; so this would give 1GB
+--   for an 8-core machine), which gives us a range of 2,000 to 18,000 here.
+--     Analysis of telemetry is hazy here; see 
+--   https://github.com/hasura/graphql-engine/issues/5363 for some discussion.
 planCacheSizeEnv :: (String, String)
 planCacheSizeEnv =
   ( "HASURA_GRAPHQL_QUERY_PLAN_CACHE_SIZE"
   , "The maximum number of query plans that can be cached, allowed values: 0-65535, " <>
-    "0 disables the cache. If this value is not set, there is no limit on the number " <>
-    "of plans that are cached"
+    "0 disables the cache. Default 4000"
   )
 
 parsePlanCacheSize :: Parser (Maybe Cache.CacheSize)

server/src-lib/Hasura/Server/Init/Config.hs

@@ -13,7 +13,7 @@ import           Data.Char                        (toLower)
 import           Data.Time
 import           Network.Wai.Handler.Warp         (HostPreference)
 
-import qualified Hasura.Cache                     as Cache
+import qualified Hasura.Cache.Bounded             as Cache
 import qualified Hasura.GraphQL.Execute           as E
 import qualified Hasura.GraphQL.Execute.LiveQuery as LQ
 import qualified Hasura.Logging                   as L

server/src-test/Hasura/CacheBoundedSpec.hs

@@ -16,7 +16,7 @@ spec = describe "Bounded cache data structure" $ do
           _ -> error "stripes wrong"
 
   it "works for 0 size" $ do
-    c <- Cache.initialise 1 0
+    c <- Cache.initialiseStripes 1 0
     Cache.lookup 'X' c `shouldReturn` Nothing
     Cache.insert 'Y' 'Y' c
     Cache.lookup 'Y' c `shouldReturn` Nothing
@@ -24,7 +24,7 @@ spec = describe "Bounded cache data structure" $ do
 
   -- basic functionality check:
   it "seems to be working right" $ do
-    c <- Cache.initialise 1 3
+    c <- Cache.initialiseStripes 1 3
 
     Cache.insert 'A' 'A' c
     checkEntries c [('A', 0, 'A')]