on master help sign in
Pick metadata parsing from old develop
commit89f324f81b40d6818e6307794fe06b60053adbc0 authorJasper Van der Jeugt <m@jaspervdj.be> dateThu 8 Nov 2012, 12:45 (14 years ago) parente5c97d9
13 files changed+575−197
hakyll.cabal+4−2 src/Hakyll/Core/Compiler.hs+2−3 src/Hakyll/Core/Resource.hs+30−10 src/Hakyll/Core/Resource/Metadata.hs+118 src/Hakyll/Core/Resource/MetadataCache.hs+61 src/Hakyll/Core/Resource/Modified.hs+82 src/Hakyll/Core/Resource/Pattern.hs+160 src/Hakyll/Core/Resource/Provider.hs+29−109 src/Hakyll/Core/Resource/Provider/Dummy.hs−25 src/Hakyll/Core/Resource/Provider/File.hs−39 src/Hakyll/Core/Resource/Provider/Internal.hs+84 src/Hakyll/Core/Run.hs+4−6 src/Hakyll/Core/Util/File.hs+1−3
hakyll.cabal+4−2
@@ -101,8 +101,10 @@ Library
101101 Hakyll.Core.Logger
102102 Hakyll.Core.Resource
103103 Hakyll.Core.Resource.Provider
104− Hakyll.Core.Resource.Provider.Dummy
105− Hakyll.Core.Resource.Provider.File
104+ Hakyll.Core.Resource.Provider.Internal
105+ Hakyll.Core.Resource.Metadata
106+ Hakyll.Core.Resource.MetadataCache
107+ Hakyll.Core.Resource.Modified
106108 Hakyll.Core.Routes
107109 Hakyll.Core.Rules
108110 Hakyll.Core.Run
src/Hakyll/Core/Compiler.hs+2−3
@@ -210,13 +210,12 @@ getResourceLBS = getResourceWith resourceLBS
210210
211211-- | Overloadable function for 'getResourceString' and 'getResourceLBS'
212212--
213−getResourceWith :: (ResourceProvider -> Resource -> IO a)
214− -> Compiler Resource a
213+getResourceWith :: (Resource -> IO a) -> Compiler Resource a
215214getResourceWith reader = fromJob $ \r -> CompilerM $ do
216215 let filePath = unResource r
217216 provider <- compilerResourceProvider <$> ask
218217 if resourceExists provider r
219− then liftIO $ reader provider r
218+ then liftIO $ reader r
220219 else throwError $ error' filePath
221220 where
222221 error' id' = "Hakyll.Core.Compiler.getResourceWith: resource "
src/Hakyll/Core/Resource.hs+30−10
@@ -1,31 +1,51 @@
1+--------------------------------------------------------------------------------
12-- | Module exporting the simple 'Resource' type
2−--
33module Hakyll.Core.Resource
4− ( Resource
5− , unResource
4+ ( -- * Constructing and deconstructing resources
5+ Resource
66 , resource
7+ , unResource
8+
9+ -- * Conversions to and from identifiers
710 , fromIdentifier
811 , toIdentifier
12+
13+ -- * TODO: Move me
14+ , Metadata
915 ) where
1016
11−import Hakyll.Core.Identifier
1217
18+--------------------------------------------------------------------------------
19+import Data.Map (Map)
20+
21+
22+--------------------------------------------------------------------------------
23+import Hakyll.Core.Identifier
24+
25+
26+--------------------------------------------------------------------------------
1327-- | A resource
14−--
1528newtype Resource = Resource {unResource :: FilePath}
1629 deriving (Eq, Show, Ord)
1730
31+
32+--------------------------------------------------------------------------------
1833-- | Smart constructor to ensure we have @/@ as path separator
19−--
2034resource :: FilePath -> Resource
2135resource = fromIdentifier . parseIdentifier
2236
23−-- | Create a resource from an identifier
24−--
37+
38+--------------------------------------------------------------------------------
39+-- | Find the resource for an identifier
2540fromIdentifier :: Identifier a -> Resource
2641fromIdentifier = Resource . toFilePath
2742
28−-- | Map the resource to an identifier. Note that the group will not be set!
29−--
43+
44+--------------------------------------------------------------------------------
45+-- | Convert a resource to an identifier
3046toIdentifier :: Resource -> Identifier a
3147toIdentifier = parseIdentifier . unResource
48+
49+
50+--------------------------------------------------------------------------------
51+type Metadata = Map String String
src/Hakyll/Core/Resource/Metadata.hsnew file+118
@@ -0,0 +1,118 @@
1+--------------------------------------------------------------------------------
2+-- | Internal module to parse metadata
3+module Hakyll.Core.Resource.Metadata
4+ ( loadMetadata
5+ ) where
6+
7+
8+--------------------------------------------------------------------------------
9+import Control.Applicative ((<$>), (<*), (<*>))
10+import Control.Arrow (second)
11+import qualified Data.ByteString.Char8 as BC
12+import qualified Data.Map as M
13+import System.IO as IO
14+import Text.Parsec ((<?>))
15+import qualified Text.Parsec as P
16+import Text.Parsec.String (Parser)
17+
18+
19+--------------------------------------------------------------------------------
20+import Hakyll.Core.Resource
21+import Hakyll.Core.Resource.Provider.Internal
22+import Hakyll.Core.Util.String
23+
24+
25+--------------------------------------------------------------------------------
26+loadMetadata :: ResourceProvider -> Resource -> IO (Metadata, Maybe String)
27+loadMetadata rp r = do
28+ hasHeader <- probablyHasMetadataHeader fp
29+ (md, body) <- if hasHeader
30+ then second Just <$> loadMetadataHeader fp
31+ else return (M.empty, Nothing)
32+
33+ emd <- if resourceExists rp mr then loadMetadataFile mfp else return M.empty
34+
35+ return (M.union md emd, body)
36+ where
37+ fp = unResource r
38+ mr = resourceMetadataResource r
39+ mfp = unResource mr
40+
41+
42+--------------------------------------------------------------------------------
43+loadMetadataHeader :: FilePath -> IO (Metadata, String)
44+loadMetadataHeader fp = do
45+ contents <- readFile fp
46+ case P.parse page fp contents of
47+ Left err -> error (show err)
48+ Right (md, b) -> return (M.fromList md, b)
49+
50+
51+--------------------------------------------------------------------------------
52+loadMetadataFile :: FilePath -> IO Metadata
53+loadMetadataFile fp = do
54+ contents <- readFile fp
55+ case P.parse metadata fp contents of
56+ Left err -> error (show err)
57+ Right md -> return $ M.fromList md
58+
59+
60+--------------------------------------------------------------------------------
61+-- | Check if a file "probably" has a metadata header. The main goal of this is
62+-- to exclude binary files (which are unlikely to start with "---").
63+probablyHasMetadataHeader :: FilePath -> IO Bool
64+probablyHasMetadataHeader fp = do
65+ handle <- IO.openFile fp IO.ReadMode
66+ bs <- BC.hGet handle 1024
67+ IO.hClose handle
68+ return $ isMetadataHeader bs
69+ where
70+ isMetadataHeader bs =
71+ let pre = BC.takeWhile (\x -> x /= '\n' && x /= '\r') bs
72+ in BC.length pre >= 3 && BC.all (== '-') pre
73+
74+
75+--------------------------------------------------------------------------------
76+-- | Space or tab, no newline
77+inlineSpace :: Parser Char
78+inlineSpace = P.oneOf ['\t', ' '] <?> "space"
79+
80+
81+--------------------------------------------------------------------------------
82+-- | Parse a single metadata field
83+metadataField :: Parser (String, String)
84+metadataField = do
85+ key <- P.manyTill P.alphaNum $ P.char ':'
86+ P.skipMany1 inlineSpace <?> "space followed by metadata for: " ++ key
87+ value <- P.manyTill P.anyChar P.newline
88+ trailing' <- P.many trailing
89+ return (key, trim $ value ++ concat trailing')
90+ where
91+ trailing = (++) <$> P.many1 inlineSpace <*> P.manyTill P.anyChar P.newline
92+
93+
94+--------------------------------------------------------------------------------
95+-- | Parse a metadata block
96+metadata :: Parser [(String, String)]
97+metadata = P.many metadataField
98+
99+
100+--------------------------------------------------------------------------------
101+-- | Parse a metadata block, including delimiters and trailing newlines
102+metadataBlock :: Parser [(String, String)]
103+metadataBlock = do
104+ open <- P.many1 (P.char '-') <* P.many inlineSpace <* P.newline
105+ metadata' <- metadata
106+ _ <- P.choice $ map (P.string . replicate (length open)) ['-', '.']
107+ P.skipMany inlineSpace
108+ P.skipMany1 P.newline
109+ return metadata'
110+
111+
112+--------------------------------------------------------------------------------
113+-- | Parse a page consisting of a metadata header and a body
114+page :: Parser ([(String, String)], String)
115+page = do
116+ metadata' <- P.option [] metadataBlock
117+ body <- P.many P.anyChar
118+ return (metadata', body)
src/Hakyll/Core/Resource/MetadataCache.hsnew file+61
@@ -0,0 +1,61 @@
1+--------------------------------------------------------------------------------
2+module Hakyll.Core.Resource.MetadataCache
3+ ( resourceMetadata
4+ , resourceBody
5+ , resourceInvalidateMetadataCache
6+ ) where
7+
8+
9+--------------------------------------------------------------------------------
10+import Hakyll.Core.Resource
11+import Hakyll.Core.Resource.Metadata
12+import Hakyll.Core.Resource.Provider.Internal
13+import qualified Hakyll.Core.Store as Store
14+
15+
16+--------------------------------------------------------------------------------
17+resourceMetadata :: ResourceProvider -> Resource -> IO Metadata
18+resourceMetadata rp r = do
19+ load rp r
20+ Store.Found md <- Store.get (resourceStore rp)
21+ [name, unResource r, "metadata"]
22+ return md
23+
24+
25+--------------------------------------------------------------------------------
26+resourceBody :: ResourceProvider -> Resource -> IO String
27+resourceBody rp r = do
28+ load rp r
29+ Store.Found bd <- Store.get (resourceStore rp)
30+ [name, unResource r, "body"]
31+ maybe (resourceString r) return bd
32+
33+
34+--------------------------------------------------------------------------------
35+resourceInvalidateMetadataCache :: ResourceProvider -> Resource -> IO ()
36+resourceInvalidateMetadataCache rp r = do
37+ Store.delete (resourceStore rp) [name, unResource r, "metadata"]
38+ Store.delete (resourceStore rp) [name, unResource r, "body"]
39+
40+
41+--------------------------------------------------------------------------------
42+load :: ResourceProvider -> Resource -> IO ()
43+load rp r = do
44+ mmd <- Store.get store mdk :: IO (Store.Result Metadata)
45+ case mmd of
46+ -- Already loaded
47+ Store.Found _ -> return ()
48+ -- Not yet loaded
49+ _ -> do
50+ (metadata, body) <- loadMetadata rp r
51+ Store.set store mdk metadata
52+ Store.set store bk body
53+ where
54+ store = resourceStore rp
55+ mdk = [name, unResource r, "metadata"]
56+ bk = [name, unResource r, "body"]
57+
58+
59+--------------------------------------------------------------------------------
60+name :: String
61+name = "Hakyll.Core.Resource.Provider.MetadataCache"
src/Hakyll/Core/Resource/Modified.hsnew file+82
@@ -0,0 +1,82 @@
1+--------------------------------------------------------------------------------
2+module Hakyll.Core.Resource.Modified
3+ ( resourceModified
4+ , resourceModificationTime
5+ ) where
6+
7+
8+--------------------------------------------------------------------------------
9+import Control.Applicative ((<$>), (<*>))
10+import Control.Monad (when)
11+import qualified Crypto.Hash.MD5 as MD5
12+import qualified Data.ByteString as B
13+import qualified Data.ByteString.Lazy as BL
14+import Data.IORef
15+import qualified Data.Map as M
16+import Data.Time (UTCTime)
17+import System.Directory (getModificationTime)
18+
19+
20+--------------------------------------------------------------------------------
21+import Hakyll.Core.Resource
22+import Hakyll.Core.Resource.MetadataCache
23+import Hakyll.Core.Resource.Provider.Internal
24+import Hakyll.Core.Store (Store)
25+import qualified Hakyll.Core.Store as Store
26+
27+
28+--------------------------------------------------------------------------------
29+-- | A resource is modified if it or its metadata has changed
30+resourceModified :: ResourceProvider -> Resource -> IO Bool
31+resourceModified rp r
32+ | not exists = return False
33+ | otherwise = do
34+ cache <- readIORef cacheRef
35+ case M.lookup r cache of
36+ Just m -> return m
37+ Nothing -> do
38+ -- Check if the actual file was modified, and do a recursive
39+ -- call to check if the metadata file was modified
40+ m <- (||)
41+ <$> fileDigestModified store (unResource r)
42+ <*> resourceModified rp (resourceMetadataResource r)
43+ modifyIORef cacheRef (M.insert r m)
44+
45+ -- Important!
46+ when m $ resourceInvalidateMetadataCache rp r
47+
48+ return m
49+ where
50+ exists = resourceExists rp r
51+ store = resourceStore rp
52+ cacheRef = resourceModifiedCache rp
53+
54+
55+--------------------------------------------------------------------------------
56+-- | Utility: Check if a the digest of a file was modified
57+fileDigestModified :: Store -> FilePath -> IO Bool
58+fileDigestModified store fp = do
59+ -- Get the latest seen digest from the store, and calculate the current
60+ -- digest for the
61+ lastDigest <- Store.get store key
62+ newDigest <- fileDigest fp
63+ if Store.Found newDigest == lastDigest
64+ -- All is fine, not modified
65+ then return False
66+ -- Resource modified; store new digest
67+ else do
68+ Store.set store key newDigest
69+ return True
70+ where
71+ key = ["Hakyll.Core.Resource.Provider.fileModified", fp]
72+
73+
74+--------------------------------------------------------------------------------
75+-- | Utility: Retrieve a digest for a given file
76+fileDigest :: FilePath -> IO B.ByteString
77+fileDigest = fmap MD5.hashlazy . BL.readFile
78+
79+
80+--------------------------------------------------------------------------------
81+resourceModificationTime :: Resource -> IO UTCTime
82+resourceModificationTime = getModificationTime . unResource
src/Hakyll/Core/Resource/Pattern.hsnew file+160
@@ -0,0 +1,160 @@
1+--------------------------------------------------------------------------------
2+-- | Module providing pattern matching and capturing on file names.
3+-- 'Pattern's come in two kinds:
4+--
5+-- * Simple glob patterns, like @foo\/*@;
6+--
7+-- * Custom, arbitrary predicates of the type @Identifier -> Bool@.
8+--
9+-- They both have advantages and disadvantages. By default, globs are used,
10+-- unless you construct your 'Pattern' using the 'predicate' function.
11+--
12+-- A very simple pattern could be, for example, @foo\/bar@. This pattern will
13+-- only match the exact @foo\/bar@ identifier.
14+--
15+-- To match more than one identifier, there are different captures that one can
16+-- use:
17+--
18+-- * @*@: matches at most one element of an identifier;
19+--
20+-- * @**@: matches one or more elements of an identifier.
21+--
22+-- Some examples:
23+--
24+-- * @foo\/*@ will match @foo\/bar@ and @foo\/foo@, but not @foo\/bar\/qux@;
25+--
26+-- * @**@ will match any identifier;
27+--
28+-- * @foo\/**@ will match @foo\/bar@ and @foo\/bar\/qux@, but not @bar\/foo@;
29+--
30+-- * @foo\/*.html@ will match all HTML files in the @foo\/@ directory.
31+--
32+-- The 'capture' function allows the user to get access to the elements captured
33+-- by the capture elements in the pattern.
34+--
35+-- Like an 'Identifier', a 'Pattern' also has a type parameter. This is simply
36+-- an extra layer of safety, and can be discarded using the 'castPattern'
37+-- function.
38+module Hakyll.Core.Resource.Pattern
39+ ( Pattern
40+ , parsePattern
41+ , capture
42+ ) where
43+
44+
45+--------------------------------------------------------------------------------
46+import Control.Arrow ((&&&), (>>>))
47+import Control.Monad (msum)
48+import Data.List (inits, isPrefixOf, tails)
49+import GHC.Exts (IsString, fromString)
50+
51+
52+--------------------------------------------------------------------------------
53+import Hakyll.Core.Resource
54+
55+
56+--------------------------------------------------------------------------------
57+-- | One base element of a pattern
58+data GlobComponent
59+ = Capture
60+ | CaptureMany
61+ | Literal String
62+ deriving (Eq, Show)
63+
64+
65+--------------------------------------------------------------------------------
66+-- | Type that allows matching on identifiers
67+newtype Pattern = Pattern [GlobComponent]
68+ deriving (Show)
69+
70+
71+--------------------------------------------------------------------------------
72+instance IsString Pattern where
73+ fromString = parsePattern
74+
75+
76+--------------------------------------------------------------------------------
77+-- | Parse a pattern from a string
78+parsePattern :: String -> Pattern
79+parsePattern = Pattern . parse
80+ where
81+ parse str =
82+ let (chunk, rest) = break (`elem` "\\*") str
83+ in case rest of
84+ ('\\' : x : xs) -> Literal (chunk ++ [x]) : parse xs
85+ ('*' : '*' : xs) -> Literal chunk : CaptureMany : parse xs
86+ ('*' : xs) -> Literal chunk : Capture : parse xs
87+ xs -> Literal chunk : Literal xs : []
88+
89+
90+--------------------------------------------------------------------------------
91+-- | Split a list at every possible point, generate a list of (init, tail)
92+-- cases. The result is sorted with inits decreasing in length.
93+splits :: [a] -> [([a], [a])]
94+splits = inits &&& tails >>> uncurry zip >>> reverse
95+
96+
97+--------------------------------------------------------------------------------
98+-- | Match a glob against a pattern, generating a list of captures
99+capture :: Pattern -> Resource -> Maybe [String]
100+capture (Pattern p) rs = capture' p (unResource rs)
101+
102+
103+--------------------------------------------------------------------------------
104+-- | Internal verion of 'capture'
105+capture' :: [GlobComponent] -> String -> Maybe [String]
106+capture' [] [] = Just [] -- An empty match
107+capture' [] _ = Nothing -- No match
108+capture' (Literal l : ms) str
109+ -- Match the literal against the string
110+ | l `isPrefixOf` str = capture' ms $ drop (length l) str
111+ | otherwise = Nothing
112+capture' (Capture : ms) str =
113+ -- Match until the next /
114+ let (chunk, rest) = break (== '/') str
115+ in msum $ [ fmap (i :) (capture' ms (t ++ rest)) | (i, t) <- splits chunk ]
116+capture' (CaptureMany : ms) str =
117+ -- Match everything
118+ msum $ [ fmap (i :) (capture' ms t) | (i, t) <- splits str ]
119+
120+
121+--------------------------------------------------------------------------------
122+-- | Create an identifier from a pattern by filling in the captures with a given
123+-- string
124+--
125+-- Example:
126+--
127+-- > fromCapture (parsePattern "tags/*") "foo"
128+--
129+-- Result:
130+--
131+-- > "tags/foo"
132+{-
133+fromCapture :: Pattern -> String -> Identifier
134+fromCapture pattern = fromCaptures pattern . repeat
135+-}
136+
137+
138+--------------------------------------------------------------------------------
139+-- | Create an identifier from a pattern by filling in the captures with the
140+-- given list of strings
141+--
142+{-
143+fromCaptures :: Pattern -> [String] -> String
144+fromCaptures (Pattern p) = fromCaptures' p
145+-}
146+
147+
148+--------------------------------------------------------------------------------
149+-- | Internally used version of 'fromCaptures'
150+{-
151+fromCaptures' :: [GlobComponent] -> [String] -> String
152+fromCaptures' [] _ = mempty
153+fromCaptures' (m : ms) [] = case m of
154+ Literal l -> l `mappend` fromCaptures' ms []
155+ _ -> error $ "Hakyll.Core.Identifier.Pattern.fromCaptures': "
156+ ++ "identifier list exhausted"
157+fromCaptures' (m : ms) ids@(i : is) = case m of
158+ Literal l -> l `mappend` fromCaptures' ms ids
159+ _ -> i `mappend` fromCaptures' ms is
160+-}
src/Hakyll/Core/Resource/Provider.hs+29−109
@@ -1,125 +1,45 @@
11--------------------------------------------------------------------------------
2−-- | This module provides an API for resource providers. Resource providers
3−-- allow Hakyll to get content from resources; the type of resource depends on
4−-- the concrete instance.
5−--
6−-- A resource is represented by the 'Resource' type. This is basically just a
7−-- newtype wrapper around 'Identifier' -- but it has an important effect: it
8−-- guarantees that a resource with this identifier can be provided by one or
9−-- more resource providers.
10−--
11−-- Therefore, it is not recommended to read files directly -- you should use the
12−-- provided 'Resource' methods.
13−--
2+-- | This module provides an wrapper API around the file system which does some
3+-- caching.
144module Hakyll.Core.Resource.Provider
15− ( ResourceProvider (..)
5+ ( -- * Constructing resource providers
6+ ResourceProvider
7+ , newResourceProvider
8+
9+ -- * Querying resource properties
1610 , resourceList
17− , makeResourceProvider
1811 , resourceExists
19− , resourceDigest
2012 , resourceModified
21− ) where
22−
23−
24−--------------------------------------------------------------------------------
25−import Control.Applicative ((<$>))
26−import Control.Concurrent (MVar, readMVar, modifyMVar_, newMVar)
27−import Data.Map (Map)
28−import qualified Data.Map as M
29−import qualified Data.Set as S
30−
31−
32−--------------------------------------------------------------------------------
33−import Data.Time (UTCTime)
34−import qualified Crypto.Hash.MD5 as MD5
35−import qualified Data.ByteString as B
36−import qualified Data.ByteString.Lazy as LB
37−
38−
39−--------------------------------------------------------------------------------
40−import Hakyll.Core.Store (Store)
41−import Hakyll.Core.Resource
42−import qualified Hakyll.Core.Store as Store
13+ , resourceModificationTime
4314
15+ -- * Access to raw resource content
16+ , resourceString
17+ , resourceLBS
4418
45−--------------------------------------------------------------------------------
46−-- | A value responsible for retrieving and listing resources
47−data ResourceProvider = ResourceProvider
48− { -- | A set of all resources this provider is able to provide
49− resourceSet :: S.Set Resource
50− , -- | Retrieve a certain resource as string
51− resourceString :: Resource -> IO String
52− , -- | Retrieve a certain resource as lazy bytestring
53− resourceLBS :: Resource -> IO LB.ByteString
54− , -- | Check when a resource was last modified
55− resourceModificationTime :: Resource -> IO UTCTime
56− , -- | Cache keeping track of modified items
57− resourceModifiedCache :: MVar (Map Resource Bool)
58− }
59−
60−
61−--------------------------------------------------------------------------------
62−-- | Create a resource provider
63−makeResourceProvider :: [Resource] -- ^ Resource list
64− -> (Resource -> IO String) -- ^ String reader
65− -> (Resource -> IO LB.ByteString) -- ^ ByteString reader
66− -> (Resource -> IO UTCTime) -- ^ Time checker
67− -> IO ResourceProvider -- ^ Resulting provider
68−makeResourceProvider l s b t =
69− ResourceProvider (S.fromList l) s b t <$> newMVar M.empty
70−
71−
72−--------------------------------------------------------------------------------
73−-- | Get the list of all resources
74−resourceList :: ResourceProvider -> [Resource]
75−resourceList = S.toList . resourceSet
76−
77−
78−--------------------------------------------------------------------------------
79−-- | Check if a given identifier has a resource
80−resourceExists :: ResourceProvider -> Resource -> Bool
81−resourceExists provider = flip S.member $ resourceSet provider
19+ -- * Access to metadata and body content
20+ , resourceMetadata
21+ , resourceBody
22+ ) where
8223
8324
8425--------------------------------------------------------------------------------
85−-- | Retrieve a digest for a given resource
86−resourceDigest :: ResourceProvider -> Resource -> IO B.ByteString
87−resourceDigest provider = fmap MD5.hashlazy . resourceLBS provider
26+import Hakyll.Core.Resource
27+import qualified Hakyll.Core.Resource.MetadataCache as Internal
28+import Hakyll.Core.Resource.Modified
29+import Hakyll.Core.Resource.Provider.Internal
8830
8931
9032--------------------------------------------------------------------------------
91−-- | Check if a resource was modified
92−resourceModified :: ResourceProvider -> Store -> Resource -> IO Bool
93−resourceModified provider store r = do
94− cache <- readMVar mvar
95− case M.lookup r cache of
96− -- Already in the cache
97− Just m -> return m
98− -- Not yet in the cache, check digests (if it exists)
99− Nothing -> do
100− m <- if resourceExists provider r
101− then digestModified provider store r
102− else return False
103− modifyMVar_ mvar (return . M.insert r m)
104− return m
105− where
106− mvar = resourceModifiedCache provider
33+-- | Wrapper to ensure metadata cache is invalidated if necessary
34+resourceMetadata :: ResourceProvider -> Resource -> IO Metadata
35+resourceMetadata rp r = do
36+ _ <- resourceModified rp r
37+ Internal.resourceMetadata rp r
10738
10839
10940--------------------------------------------------------------------------------
110−-- | Check if a resource digest was modified
111−digestModified :: ResourceProvider -> Store -> Resource -> IO Bool
112−digestModified provider store r = do
113− -- Get the latest seen digest from the store
114− lastDigest <- Store.get store key
115− -- Calculate the digest for the resource
116− newDigest <- resourceDigest provider r
117− -- Check digests
118− if Store.Found newDigest == lastDigest
119− -- All is fine, not modified
120− then return False
121− -- Resource modified; store new digest
122− else do Store.set store key newDigest
123− return True
124− where
125− key = ["Hakyll.Core.ResourceProvider.digestModified", unResource r]
41+-- | Wrapper to ensure metadata cache is invalidated if necessary
42+resourceBody :: ResourceProvider -> Resource -> IO String
43+resourceBody rp r = do
44+ _ <- resourceModified rp r
45+ Internal.resourceBody rp r
src/Hakyll/Core/Resource/Provider/Dummy.hsdeleted−25
@@ -1,25 +0,0 @@
1−-- | Dummy resource provider for testing purposes
2−--
3−module Hakyll.Core.Resource.Provider.Dummy
4− ( dummyResourceProvider
5− ) where
6−
7−import Data.Map (Map)
8−import qualified Data.Map as M
9−
10−import Data.Time (getCurrentTime)
11−import Data.ByteString.Lazy (ByteString)
12−import qualified Data.Text.Lazy as TL
13−import qualified Data.Text.Lazy.Encoding as TL
14−
15−import Hakyll.Core.Resource
16−import Hakyll.Core.Resource.Provider
17−
18−-- | Create a dummy 'ResourceProvider'
19−--
20−dummyResourceProvider :: Map String ByteString -> IO ResourceProvider
21−dummyResourceProvider vfs = makeResourceProvider
22− (map resource (M.keys vfs))
23− (return . TL.unpack . TL.decodeUtf8 . (vfs M.!) . unResource)
24− (return . (vfs M.!) . unResource)
25− (const getCurrentTime)
src/Hakyll/Core/Resource/Provider/File.hsdeleted−39
@@ -1,39 +0,0 @@
1−-- | A concrete 'ResourceProvider' that gets it's resources from the filesystem
2−--
3−{-# LANGUAGE CPP #-}
4−module Hakyll.Core.Resource.Provider.File
5− ( fileResourceProvider
6− ) where
7−
8−import Control.Applicative ((<$>))
9−
10−import Data.Time (readTime)
11−import System.Directory (getModificationTime)
12−import System.Locale (defaultTimeLocale)
13−import System.Time (formatCalendarTime, toCalendarTime)
14−import qualified Data.ByteString.Lazy as LB
15−
16−import Hakyll.Core.Resource
17−import Hakyll.Core.Resource.Provider
18−import Hakyll.Core.Util.File
19−import Hakyll.Core.Configuration
20−
21−-- | Create a filesystem-based 'ResourceProvider'
22−--
23−fileResourceProvider :: HakyllConfiguration -> IO ResourceProvider
24−fileResourceProvider configuration = do
25− -- Retrieve a list of paths
26− list <- map resource . filter (not . shouldIgnoreFile configuration) <$>
27− getRecursiveContents False "."
28− makeResourceProvider list (readFile . unResource)
29− (LB.readFile . unResource)
30− (mtime . unResource)
31− where
32− mtime r = do
33−#if MIN_VERSION_directory(1,2,0)
34− getModificationTime r
35−#else
36− ct <- toCalendarTime =<< getModificationTime r
37− let str = formatCalendarTime defaultTimeLocale "%s" ct
38− return $ readTime defaultTimeLocale "%s" str
39−#endif
src/Hakyll/Core/Resource/Provider/Internal.hsnew file+84
@@ -0,0 +1,84 @@
1+--------------------------------------------------------------------------------
2+module Hakyll.Core.Resource.Provider.Internal
3+ ( ResourceProvider (..)
4+ , newResourceProvider
5+
6+ , resourceList
7+ , resourceExists
8+ , resourceMetadataResource
9+
10+ , resourceString
11+ , resourceLBS
12+ ) where
13+
14+
15+--------------------------------------------------------------------------------
16+import Control.Applicative ((<$>))
17+import qualified Data.ByteString.Lazy as BL
18+import Data.IORef
19+import Data.Map (Map)
20+import qualified Data.Map as M
21+import Data.Set (Set)
22+import qualified Data.Set as S
23+import System.FilePath (addExtension)
24+
25+
26+--------------------------------------------------------------------------------
27+import Hakyll.Core.Resource
28+import Hakyll.Core.Store
29+import Hakyll.Core.Util.File
30+
31+
32+--------------------------------------------------------------------------------
33+-- | Responsible for retrieving and listing resources
34+data ResourceProvider = ResourceProvider
35+ { -- | A list of all files found
36+ resourceSet :: Set Resource
37+ , -- | Cache keeping track of modified files
38+ resourceModifiedCache :: IORef (Map Resource Bool)
39+ , -- | Underlying persistent store for caching
40+ resourceStore :: Store
41+ }
42+
43+
44+--------------------------------------------------------------------------------
45+-- | Create a resource provider
46+newResourceProvider :: Store -- ^ Store to use
47+ -> (FilePath -> Bool) -- ^ Should we ignore this file?
48+ -> FilePath -- ^ Search directory
49+ -> IO ResourceProvider -- ^ Resulting provider
50+newResourceProvider store ignore directory = do
51+ list <- map resource . filter (not . ignore) <$>
52+ getRecursiveContents False directory
53+ cache <- newIORef M.empty
54+ return $ ResourceProvider (S.fromList list) cache store
55+
56+
57+--------------------------------------------------------------------------------
58+resourceList :: ResourceProvider -> [Resource]
59+resourceList = S.toList . resourceSet
60+
61+
62+--------------------------------------------------------------------------------
63+-- | Check if a given resiyrce exists
64+resourceExists :: ResourceProvider -> Resource -> Bool
65+resourceExists provider = (`S.member` resourceSet provider)
66+
67+
68+--------------------------------------------------------------------------------
69+-- | Each resource may have an associated metadata resource (with a @.metadata@
70+-- filename)
71+resourceMetadataResource :: Resource -> Resource
72+resourceMetadataResource = resource . flip addExtension "metadata" . unResource
73+
74+
75+--------------------------------------------------------------------------------
76+-- | Get the raw body of a resource as string
77+resourceString :: Resource -> IO String
78+resourceString = readFile . unResource
79+
80+
81+--------------------------------------------------------------------------------
82+-- | Get the raw body of a resource of a lazy bytestring
83+resourceLBS :: Resource -> IO BL.ByteString
84+resourceLBS = BL.readFile . unResource
src/Hakyll/Core/Run.hs+4−6
@@ -27,7 +27,6 @@ import Hakyll.Core.Identifier
2727import Hakyll.Core.Logger
2828import Hakyll.Core.Resource
2929import Hakyll.Core.Resource.Provider
30−import Hakyll.Core.Resource.Provider.File
3130import Hakyll.Core.Routes
3231import Hakyll.Core.Rules.Internal
3332import Hakyll.Core.Store (Store)
@@ -44,8 +43,8 @@ run configuration rules = do
4443 section logger "Initialising"
4544 store <- timed logger "Creating store" $
4645 Store.new (inMemoryCache configuration) $ storeDirectory configuration
47− provider <- timed logger "Creating provider" $
48− fileResourceProvider configuration
46+ provider <- timed logger "Creating provider" $ newResourceProvider
47+ store (ignoreFile configuration) "."
4948
5049 -- Fetch the old graph from the store. If we don't find it, we consider this
5150 -- to be the first run
@@ -114,7 +113,6 @@ addNewCompilers newCompilers = Runtime $ do
114113 logger <- hakyllLogger <$> ask
115114 section logger "Adding new compilers"
116115 provider <- hakyllResourceProvider <$> ask
117− store <- hakyllStore <$> ask
118116 firstRun <- hakyllFirstRun <$> ask
119117
120118 -- Old state information
@@ -134,7 +132,7 @@ addNewCompilers newCompilers = Runtime $ do
134132
135133 -- Check which items have been modified
136134 modified <- fmap S.fromList $ flip filterM (map fst newCompilers) $
137− liftIO . resourceModified provider store . fromIdentifier
135+ liftIO . resourceModified provider . fromIdentifier
138136 let checkModified = if firstRun then const True else (`S.member` modified)
139137
140138 -- Create a new analyzer and append it to the currect one
@@ -185,7 +183,7 @@ build id' = Runtime $ do
185183 let compiler = compilers M.! id'
186184
187185 -- Check if the resource was modified
188− isModified <- liftIO $ resourceModified provider store $ fromIdentifier id'
186+ isModified <- liftIO $ resourceModified provider $ fromIdentifier id'
189187
190188 -- Run the compiler
191189 result <- timed logger "Total compile time" $ liftIO $
src/Hakyll/Core/Util/File.hs+1−3
@@ -24,9 +24,7 @@ import Hakyll.Core.Configuration
2424makeDirectories :: FilePath -> IO ()
2525makeDirectories = createDirectoryIfMissing True . takeDirectory
2626
27−-- | Get all contents of a directory. Note that files starting with a dot (.)
28−-- will be ignored.
29−--
27+-- | Get all contents of a directory.
3028getRecursiveContents :: Bool -- ^ Include directories?
3129 -> FilePath -- ^ Directory to search
3230 -> IO [FilePath] -- ^ List of files found