on master help sign in
Add in-memory map to store

This allows us to get rid of the dependency lookup map and use one uniform cache/lookup.

commit77c7d8dc17a86640180b9b233f6e0fd9008c6848 authorJasper Van der Jeugt <jaspervdj@gmail.com> dateTue 4 Jan 2011, 13:09 (16 years ago) parent0969fe4
5 files changed+61−41
src/Hakyll/Core/Compiler.hs+9−15 src/Hakyll/Core/Compiler/Internal.hs+1−11 src/Hakyll/Core/Run.hs+3−7 src/Hakyll/Core/Store.hs+43−8 src/Hakyll/Core/Writable.hs+5
src/Hakyll/Core/Compiler.hs+9−15
@@ -37,15 +37,14 @@ import Hakyll.Core.Store
3737runCompiler :: Compiler () CompiledItem -- ^ Compiler to run
3838 -> Identifier -- ^ Target identifier
3939 -> ResourceProvider -- ^ Resource provider
40− -> DependencyLookup -- ^ Dependency lookup table
4140 -> Maybe FilePath -- ^ Route
4241 -> Store -- ^ Store
4342 -> Bool -- ^ Was the resource modified?
4443 -> IO CompiledItem -- ^ Resulting item
45−runCompiler compiler identifier provider lookup' route store modified = do
44+runCompiler compiler identifier provider route store modified = do
4645 -- Run the compiler job
4746 CompiledItem result <- runCompilerJob
48− compiler identifier provider lookup' route store modified
47+ compiler identifier provider route store modified
4948
5049 -- Store a copy in the cache and return
5150 storeSet store "Hakyll.Core.Compiler.runCompiler" identifier result
@@ -72,17 +71,12 @@ getResourceString = getIdentifier >>> getResourceString'
7271
7372-- | Auxiliary: get a dependency
7473--
75−getDependencyOrResult :: (Binary a, Writable a, Typeable a)
74+getDependency :: (Binary a, Writable a, Typeable a)
7675 => Identifier -> CompilerM a
77−getDependencyOrResult identifier = CompilerM $ do
78− lookup' <- compilerDependencyLookup <$> ask
76+getDependency identifier = CompilerM $ do
7977 store <- compilerStore <$> ask
80− case lookup' identifier of
81− -- Found in the dependency lookup
82− Just r -> return $ unCompiledItem r
83− -- Not found here, try the main cache
84− Nothing -> fmap (fromMaybe error') $ liftIO $
85− storeGet store "Hakyll.Core.Compiler.runCompiler" identifier
78+ fmap (fromMaybe error') $ liftIO $
79+ storeGet store "Hakyll.Core.Compiler.runCompiler" identifier
8680 where
8781 error' = error $ "Hakyll.Core.Compiler.getDependency: "
8882 ++ show identifier
@@ -99,7 +93,7 @@ require identifier f =
9993 fromDependencies (const [identifier]) >>> fromJob require'
10094 where
10195 require' x = do
102− y <- getDependencyOrResult identifier
96+ y <- getDependency identifier
10397 return $ f x y
10498
10599-- | Require a number of targets. Using this function ensures automatic handling
@@ -115,10 +109,10 @@ requireAll pattern f =
115109 getDeps = matches pattern . resourceList
116110 requireAll' x = CompilerM $ do
117111 deps <- getDeps . compilerResourceProvider <$> ask
118− items <- mapM (unCompilerM . getDependencyOrResult) deps
112+ items <- mapM (unCompilerM . getDependency) deps
119113 return $ f x items
120114
121−cached :: (Binary a)
115+cached :: (Binary a, Typeable a, Writable a)
122116 => String
123117 -> Compiler () a
124118 -> Compiler () a
src/Hakyll/Core/Compiler/Internal.hs+1−11
@@ -3,7 +3,6 @@
33{-# LANGUAGE GeneralizedNewtypeDeriving #-}
44module Hakyll.Core.Compiler.Internal
55 ( Dependencies
6− , DependencyLookup
76 , CompilerEnvironment (..)
87 , CompilerM (..)
98 , Compiler (..)
@@ -23,7 +22,6 @@ import Control.Category (Category, (.), id)
2322import Control.Arrow (Arrow, arr, first)
2423
2524import Hakyll.Core.Identifier
26−import Hakyll.Core.CompiledItem
2725import Hakyll.Core.ResourceProvider
2826import Hakyll.Core.Store
2927
@@ -31,10 +29,6 @@ import Hakyll.Core.Store
3129--
3230type Dependencies = Set Identifier
3331
34−-- | A lookup with which we can get dependencies
35−--
36−type DependencyLookup = Identifier -> Maybe CompiledItem
37−
3832-- | Environment in which a compiler runs
3933--
4034data CompilerEnvironment = CompilerEnvironment
@@ -42,8 +36,6 @@ data CompilerEnvironment = CompilerEnvironment
4236 compilerIdentifier :: Identifier
4337 , -- | Resource provider
4438 compilerResourceProvider :: ResourceProvider
45− , -- | Dependency lookup
46− compilerDependencyLookup :: DependencyLookup
4739 , -- | Site route
4840 compilerRoute :: Maybe FilePath
4941 , -- | Compiler store
@@ -81,18 +73,16 @@ instance Arrow Compiler where
8173runCompilerJob :: Compiler () a -- ^ Compiler to run
8274 -> Identifier -- ^ Target identifier
8375 -> ResourceProvider -- ^ Resource provider
84− -> DependencyLookup -- ^ Dependency lookup table
8576 -> Maybe FilePath -- ^ Route
8677 -> Store -- ^ Store
8778 -> Bool -- ^ Was the resource modified?
8879 -> IO a
89−runCompilerJob compiler identifier provider lookup' route store modified =
80+runCompilerJob compiler identifier provider route store modified =
9081 runReaderT (unCompilerM $ compilerJob compiler ()) env
9182 where
9283 env = CompilerEnvironment
9384 { compilerIdentifier = identifier
9485 , compilerResourceProvider = provider
95− , compilerDependencyLookup = lookup'
9686 , compilerRoute = route
9787 , compilerStore = store
9888 , compilerResourceModified = modified
src/Hakyll/Core/Run.hs+3−7
@@ -80,19 +80,18 @@ hakyllWith rules provider store = do
8080 putStrLn $ show ordered
8181
8282 -- Generate all the targets in order
83− _ <- foldM (addTarget route' modified') M.empty orderedCompilers
83+ _ <- mapM (addTarget route' modified') orderedCompilers
8484
8585 putStrLn "DONE."
8686 where
87− addTarget route' modified' map' (id', comp) = do
87+ addTarget route' modified' (id', comp) = do
8888 let url = runRoute route' id'
8989
9090 -- Check if the resource was modified
9191 let isModified = id' `S.member` modified'
9292
9393 -- Run the compiler
94− compiled <- runCompiler comp id' provider (dependencyLookup map')
95− url store isModified
94+ compiled <- runCompiler comp id' provider url store isModified
9695 putStrLn $ "Generated target: " ++ show id'
9796
9897 case url of
@@ -104,9 +103,6 @@ hakyllWith rules provider store = do
104103 write path compiled
105104
106105 putStrLn ""
107− return $ M.insert id' compiled map'
108−
109− dependencyLookup map' id' = M.lookup id' map'
110106
111107-- | Return a set of modified identifiers
112108--
src/Hakyll/Core/Store.hs+43−8
@@ -7,25 +7,45 @@ module Hakyll.Core.Store
77 , storeGet
88 ) where
99
10−import Control.Applicative ((<$>))
10+import Control.Concurrent.MVar (MVar, newMVar, readMVar, modifyMVar_)
1111import System.FilePath ((</>))
1212import System.Directory (doesFileExist)
13+import Data.Map (Map)
14+import qualified Data.Map as M
1315
1416import Data.Binary (Binary, encodeFile, decodeFile)
17+import Data.Typeable (Typeable)
1518
19+import Hakyll.Core.CompiledItem
20+import Hakyll.Core.Writable
1621import Hakyll.Core.Identifier
1722import Hakyll.Core.Util.File
1823
1924-- | Data structure used for the store
2025--
2126data Store = Store
22− { storeDirectory :: FilePath
27+ { -- | All items are stored on the filesystem
28+ storeDirectory :: FilePath
29+ , -- | And some items are also kept in-memory
30+ storeMap :: MVar (Map FilePath CompiledItem)
2331 }
2432
2533-- | Initialize the store
2634--
2735makeStore :: FilePath -> IO Store
28−makeStore directory = return Store {storeDirectory = directory}
36+makeStore directory = do
37+ mvar <- newMVar M.empty
38+ return Store
39+ { storeDirectory = directory
40+ , storeMap = mvar
41+ }
42+
43+-- | Auxiliary: add an item to the map
44+--
45+addToMap :: (Binary a, Typeable a, Writable a)
46+ => Store -> FilePath -> a -> IO ()
47+addToMap store path value =
48+ modifyMVar_ (storeMap store) $ return . M.insert path (compiledItem value)
2949
3050-- | Create a path
3151--
@@ -35,19 +55,34 @@ makePath store name identifier =
3555
3656-- | Store an item
3757--
38−storeSet :: Binary a => Store -> String -> Identifier -> a -> IO ()
58+storeSet :: (Binary a, Typeable a, Writable a)
59+ => Store -> String -> Identifier -> a -> IO ()
3960storeSet store name identifier value = do
4061 makeDirectories path
4162 encodeFile path value
63+ addToMap store path value
4264 where
4365 path = makePath store name identifier
4466
4567-- | Load an item
4668--
47−storeGet :: Binary a => Store -> String -> Identifier -> IO (Maybe a)
69+storeGet :: (Binary a, Typeable a, Writable a)
70+ => Store -> String -> Identifier -> IO (Maybe a)
4871storeGet store name identifier = do
49− exists <- doesFileExist path
50− if exists then Just <$> decodeFile path
51− else return Nothing
72+ -- First check the in-memory map
73+ map' <- readMVar $ storeMap store
74+ case M.lookup path map' of
75+ -- Found in the in-memory map
76+ Just c -> return $ Just $ unCompiledItem c
77+ -- Not found in the map, try the filesystem
78+ Nothing -> do
79+ exists <- doesFileExist path
80+ if not exists
81+ -- Not found in the filesystem either
82+ then return Nothing
83+ -- Found in the filesystem
84+ else do v <- decodeFile path
85+ addToMap store path v
86+ return $ Just v
5287 where
5388 path = makePath store name identifier
src/Hakyll/Core/Writable.hs+5
@@ -8,7 +8,9 @@ module Hakyll.Core.Writable
88 ) where
99
1010import System.Directory (copyFile)
11+import Data.Word (Word8)
1112
13+import qualified Data.ByteString as SB
1214import Data.Binary (Binary)
1315import Data.Typeable (Typeable)
1416
@@ -21,6 +23,9 @@ class Writable a where
2123instance Writable [Char] where
2224 write = writeFile
2325
26+instance Writable [Word8] where
27+ write p = SB.writeFile p . SB.pack
28+
2429-- | Newtype construct around 'FilePath' which will copy the file directly
2530--
2631newtype CopyFile = CopyFile {unCopyFile :: FilePath}