on master help sign in
cleaning up HashMap and MultiMap.
commit919eb78346dfd1ee571e856faf33bfe904b6a560 authorKazu Yamamoto <kazu@iij.ad.jp> dateWed 5 Jun 2019, 11:03 (7 years ago) parent5683195
4 files changed+115−102
warp/Network/Wai/Handler/Warp/FdCache.hs+18−22 warp/Network/Wai/Handler/Warp/FileInfoCache.hs+18−21 warp/Network/Wai/Handler/Warp/HashMap.hs+24−15 warp/Network/Wai/Handler/Warp/MultiMap.hs+55−44
warp/Network/Wai/Handler/Warp/FdCache.hs+18−22
@@ -17,26 +17,24 @@ module Network.Wai.Handler.Warp.FdCache (
1717import Control.Exception (bracket)
1818import Control.Reaper
1919import Data.IORef
20−import Network.Wai.Handler.Warp.MultiMap
20+import Network.Wai.Handler.Warp.MultiMap as MM
2121import System.Posix.IO (openFd, OpenFileFlags(..), defaultFileFlags, OpenMode(ReadOnly), closeFd, FdOption(CloseOnExec), setFdOption)
2222#endif
2323import System.Posix.Types (Fd)
2424
2525----------------------------------------------------------------
2626
27−type Hash = Int
28−
2927-- | An action to activate a Fd cache entry.
3028type Refresh = IO ()
3129
32−getFdNothing :: Hash -> FilePath -> IO (Maybe Fd, Refresh)
33−getFdNothing _ _ = return (Nothing, return ())
30+getFdNothing :: FilePath -> IO (Maybe Fd, Refresh)
31+getFdNothing _ = return (Nothing, return ())
3432
3533----------------------------------------------------------------
3634
3735-- | Creating 'MutableFdCache' and executing the action in the second
3836-- argument. The first argument is a cache duration in second.
39−withFdCache :: Int -> ((Hash -> FilePath -> IO (Maybe Fd, Refresh)) -> IO a) -> IO a
37+withFdCache :: Int -> ((FilePath -> IO (Maybe Fd, Refresh)) -> IO a) -> IO a
4038#ifdef WINDOWS
4139withFdCache _ action = action getFdNothing
4240#else
@@ -65,7 +63,7 @@ inactive (MutableStatus ref) = writeIORef ref Inactive
6563
6664----------------------------------------------------------------
6765
68−data FdEntry = FdEntry !FilePath !Fd !MutableStatus
66+data FdEntry = FdEntry !Fd !MutableStatus
6967
7068openFile :: FilePath -> IO Fd
7169openFile path = do
@@ -77,25 +75,23 @@ closeFile :: Fd -> IO ()
7775closeFile = closeFd
7876
7977newFdEntry :: FilePath -> IO FdEntry
80−newFdEntry path = FdEntry path <$> openFile path <*> newActiveStatus
78+newFdEntry path = FdEntry <$> openFile path <*> newActiveStatus
8179
8280setFileCloseOnExec :: Fd -> IO ()
8381setFileCloseOnExec fd = setFdOption fd CloseOnExec True
8482
8583----------------------------------------------------------------
8684
87−type FdCache = MMap FdEntry
85+type FdCache = MultiMap FdEntry
8886
8987-- | Mutable Fd cacher.
90−newtype MutableFdCache = MutableFdCache (Reaper FdCache (Hash, FdEntry))
88+newtype MutableFdCache = MutableFdCache (Reaper FdCache (FilePath,FdEntry))
9189
9290fdCache :: MutableFdCache -> IO FdCache
9391fdCache (MutableFdCache reaper) = reaperRead reaper
9492
95−look :: MutableFdCache -> FilePath -> Hash -> IO (Maybe FdEntry)
96−look mfc path key = searchWith key check <$> fdCache mfc
97− where
98− check (FdEntry path' _ _) = path == path'
93+look :: MutableFdCache -> FilePath -> IO (Maybe FdEntry)
94+look mfc path = MM.lookup path <$> fdCache mfc
9995
10096----------------------------------------------------------------
10197
@@ -116,7 +112,7 @@ clean old = do
116112 new <- pruneWith old prune
117113 return $ merge new
118114 where
119− prune (FdEntry _ fd mst) = status mst >>= act
115+ prune (_,FdEntry fd mst) = status mst >>= act
120116 where
121117 act Active = inactive mst >> return True
122118 act Inactive = closeFd fd >> return False
@@ -126,21 +122,21 @@ clean old = do
126122terminate :: MutableFdCache -> IO ()
127123terminate (MutableFdCache reaper) = do
128124 !t <- reaperStop reaper
129− mapM_ closeIt $ toList t
125+ mapM_ (closeIt . snd) $ toList t
130126 where
131− closeIt (FdEntry _ fd _) = closeFd fd
127+ closeIt (FdEntry fd _) = closeFd fd
132128
133129----------------------------------------------------------------
134130
135131-- | Getting 'Fd' and 'Refresh' from the mutable Fd cacher.
136−getFd :: MutableFdCache -> Hash -> FilePath -> IO (Maybe Fd, Refresh)
137−getFd mfc@(MutableFdCache reaper) h path = look mfc path h >>= get
132+getFd :: MutableFdCache -> FilePath -> IO (Maybe Fd, Refresh)
133+getFd mfc@(MutableFdCache reaper) path = look mfc path >>= get
138134 where
139135 get Nothing = do
140− ent@(FdEntry _ fd mst) <- newFdEntry path
141− reaperAdd reaper (h, ent)
136+ ent@(FdEntry fd mst) <- newFdEntry path
137+ reaperAdd reaper (path,ent)
142138 return (Just fd, refresh mst)
143− get (Just (FdEntry _ fd mst)) = do
139+ get (Just (FdEntry fd mst)) = do
144140 refresh mst
145141 return (Just fd, refresh mst)
146142#endif
warp/Network/Wai/Handler/Warp/FileInfoCache.hs+18−21
@@ -2,7 +2,6 @@
22
33module Network.Wai.Handler.Warp.FileInfoCache (
44 FileInfo(..)
5− , Hash
65 , withFileInfoCache
76 , getInfo -- test purpose only
87 ) where
@@ -18,8 +17,6 @@ import Network.Wai.Handler.Warp.Imports
1817
1918----------------------------------------------------------------
2019
21−type Hash = Int
22−
2320-- | File information.
2421data FileInfo = FileInfo {
2522 fileInfoName :: !FilePath
@@ -29,8 +26,8 @@ data FileInfo = FileInfo {
2926 } deriving (Eq, Show)
3027
3128data Entry = Negative | Positive FileInfo
32−type Cache = HashMap FilePath Entry
33−type FileInfoCache = Reaper Cache (Int,FilePath,Entry)
29+type Cache = HashMap Entry
30+type FileInfoCache = Reaper Cache (FilePath,Entry)
3431
3532----------------------------------------------------------------
3633
@@ -54,29 +51,29 @@ getInfo path = do
5451 else
5552 throwIO (userError "FileInfoCache:getInfo")
5653
57−getInfoNaive :: Hash -> FilePath -> IO FileInfo
58−getInfoNaive _ = getInfo
54+getInfoNaive :: FilePath -> IO FileInfo
55+getInfoNaive = getInfo
5956
6057----------------------------------------------------------------
6158
62−getAndRegisterInfo :: FileInfoCache -> Hash -> FilePath -> IO FileInfo
63−getAndRegisterInfo reaper@Reaper{..} h path = do
59+getAndRegisterInfo :: FileInfoCache -> FilePath -> IO FileInfo
60+getAndRegisterInfo reaper@Reaper{..} path = do
6461 cache <- reaperRead
65− case M.lookup h path cache of
62+ case M.lookup path cache of
6663 Just Negative -> throwIO (userError "FileInfoCache:getAndRegisterInfo")
6764 Just (Positive x) -> return x
68− Nothing -> positive reaper h path
69− `E.onException` negative reaper h path
65+ Nothing -> positive reaper path
66+ `E.onException` negative reaper path
7067
71−positive :: FileInfoCache -> Hash -> FilePath -> IO FileInfo
72−positive Reaper{..} h path = do
68+positive :: FileInfoCache -> FilePath -> IO FileInfo
69+positive Reaper{..} path = do
7370 info <- getInfo path
74− reaperAdd (h, path, Positive info)
71+ reaperAdd (path, Positive info)
7572 return info
7673
77−negative :: FileInfoCache -> Hash -> FilePath -> IO FileInfo
78−negative Reaper{..} h path = do
79− reaperAdd (h, path,Negative)
74+negative :: FileInfoCache -> FilePath -> IO FileInfo
75+negative Reaper{..} path = do
76+ reaperAdd (path, Negative)
8077 throwIO (userError "FileInfoCache:negative")
8178
8279----------------------------------------------------------------
@@ -85,7 +82,7 @@ negative Reaper{..} h path = do
8582-- and executing the action in the second argument.
8683-- The first argument is a cache duration in second.
8784withFileInfoCache :: Int
88− -> ((Hash -> FilePath -> IO FileInfo) -> IO a)
85+ -> ((FilePath -> IO FileInfo) -> IO a)
8986 -> IO a
9087withFileInfoCache 0 action = action getInfoNaive
9188withFileInfoCache duration action =
@@ -99,8 +96,8 @@ initialize duration = mkReaper settings
9996 settings = defaultReaperSettings {
10097 reaperAction = override
10198 , reaperDelay = duration
102− , reaperCons = \(h,k,v) -> M.insert h k v
103− , reaperNull = M.null
99+ , reaperCons = \(path,v) -> M.insert path v
100+ , reaperNull = M.isEmpty
104101 , reaperEmpty = M.empty
105102 }
106103
warp/Network/Wai/Handler/Warp/HashMap.hs+24−15
@@ -1,3 +1,5 @@
1+{-# LANGUAGE BangPatterns #-}
2+
13module Network.Wai.Handler.Warp.HashMap where
24
35import Data.Hashable (hash)
@@ -7,27 +9,34 @@ import Data.Map.Strict (Map)
79import qualified Data.Map.Strict as M
810import Prelude hiding (lookup)
911
10−import Network.Wai.Handler.Warp.Imports hiding (insert, lookup)
12+----------------------------------------------------------------
1113
12−type Hash = Int
13−newtype HashMap k v = HashMap (IntMap (Map k v))
14+-- | 'HashMap' is used for cache of file information.
15+-- Hash values of file pathes are used as outer keys.
16+-- Because negative entries are also contained,
17+-- a bad guy can intentionally cause the hash collison.
18+-- So, 'Map' is used internally to prevent
19+-- the hash collision attack.
20+newtype HashMap v = HashMap (IntMap (Map FilePath v))
1421
15−hashByteString :: ByteString -> Hash
16−hashByteString = hash
22+----------------------------------------------------------------
1723
18−empty :: HashMap k v
24+empty :: HashMap v
1925empty = HashMap I.empty
2026
21−null :: HashMap k v -> Bool
22−null (HashMap hm) = I.null hm
27+isEmpty :: HashMap v -> Bool
28+isEmpty (HashMap hm) = I.null hm
29+
30+----------------------------------------------------------------
2331
24−insert :: Ord k => Hash -> k -> v -> HashMap k v -> HashMap k v
25−insert h k v (HashMap hm) = HashMap $ I.insertWith f h m hm
32+insert :: FilePath -> v -> HashMap v -> HashMap v
33+insert path v (HashMap hm) = HashMap $ I.insertWith f h m hm
2634 where
27− m = M.singleton k v
35+ !h = hash path
36+ !m = M.singleton path v
2837 f = M.union -- fimxe
29−{-# SPECIALIZE insert :: Hash -> String -> v -> HashMap String v -> HashMap String v #-}
3038
31−lookup :: Ord k => Hash -> k -> HashMap k v -> Maybe v
32−lookup h k (HashMap hm) = I.lookup h hm >>= M.lookup k
33−{-# SPECIALIZE lookup :: Hash -> String -> HashMap String v -> Maybe v #-}
39+lookup :: FilePath -> HashMap v -> Maybe v
40+lookup path (HashMap hm) = I.lookup h hm >>= M.lookup path
41+ where
42+ !h = hash path
warp/Network/Wai/Handler/Warp/MultiMap.hs+55−44
@@ -1,100 +1,111 @@
1+{-# LANGUAGE BangPatterns #-}
2+
13module Network.Wai.Handler.Warp.MultiMap (
2− MMap
4+ MultiMap
35 , isEmpty
46 , empty
57 , singleton
68 , insert
7− , search
8− , searchWith
9+ , Network.Wai.Handler.Warp.MultiMap.lookup
910 , pruneWith
1011 , toList
1112 , merge
1213 ) where
1314
15+import Data.Hashable (hash)
1416import Data.IntMap.Strict (IntMap)
1517import qualified Data.IntMap.Strict as I
16−import qualified Data.List.NonEmpty as NE
1718import Data.Semigroup
1819import Prelude -- Silence redundant import warnings
1920
20−import Network.Wai.Handler.Warp.Imports hiding ((<>), union, empty, insert)
21−
2221----------------------------------------------------------------
2322
24−type MMap v = IntMap (NonEmpty v)
23+-- | 'MultiMap' is used for cache of file descriptors.
24+-- Since multiple threads would open file descriptors for
25+-- the same file simultaneously, multiple entries must
26+-- be contained for the file.
27+-- Since hash values of file pathes are used as outer keys,
28+-- collison would happen for multiple file pathes.
29+-- Becase only positive entries are contained,
30+-- a bad guy cannot be cause the hash collision intentinally.
31+-- So, lists are good enough.
32+newtype MultiMap v = MultiMap (IntMap [(FilePath,v)])
2533
2634----------------------------------------------------------------
2735
2836-- | O(1)
29−isEmpty :: MMap v -> Bool
30−isEmpty = I.null
37+empty :: MultiMap v
38+empty = MultiMap $ I.empty
3139
3240-- | O(1)
33−empty :: MMap v
34−empty = I.empty
41+isEmpty :: MultiMap v -> Bool
42+isEmpty (MultiMap mm) = I.null mm
3543
3644----------------------------------------------------------------
3745
3846-- | O(1)
39−singleton :: Int -> v -> MMap v
40−singleton k v = I.singleton k (v :| [])
47+singleton :: FilePath -> v -> MultiMap v
48+singleton path v = MultiMap mm
49+ where
50+ !h = hash path
51+ !mm = I.singleton h [(path,v)]
4152
4253----------------------------------------------------------------
4354
44−-- | O(log n)
45−search :: Int -> MMap v -> Maybe v
46−search k m = case I.lookup k m of
55+-- | O(N)
56+lookup :: FilePath -> MultiMap v -> Maybe v
57+lookup path (MultiMap mm) = case I.lookup h mm of
4758 Nothing -> Nothing
48− Just s -> Just $! NE.head s
49−
50−-- | O(log n)
51−searchWith :: Int -> (v -> Bool) -> MMap v -> Maybe v
52−searchWith k f m = case I.lookup k m of
53− Nothing -> Nothing
54− Just nxs -> find f $ NE.toList nxs
59+ Just s -> Prelude.lookup path s
60+ where
61+ !h = hash path
5562
5663----------------------------------------------------------------
5764
5865-- | O(log n)
59−insert :: Int -> v -> MMap v -> MMap v
60−insert k v m = I.insertWith (<>) k (v :| []) m
66+insert :: FilePath -> v -> MultiMap v -> MultiMap v
67+insert path v (MultiMap mm) = MultiMap mm'
68+ where
69+ !h = hash path
70+ !mm' = I.insertWith (<>) h [(path,v)] mm
6171
6272----------------------------------------------------------------
6373
6474-- | O(n)
65−toList :: MMap v -> [v]
66−toList m = concatMap f $ I.toAscList m
67− where
68− f (_,s) = NE.toList s
75+toList :: MultiMap v -> [(FilePath,v)]
76+toList (MultiMap mm) = concatMap snd $ I.toAscList mm
6977
7078----------------------------------------------------------------
7179
7280-- | O(n)
73−pruneWith :: MMap v
74− -> (v -> IO Bool)
75− -> IO (MMap v)
76−pruneWith m action = I.fromAscList <$> go (I.toDescList m) []
81+pruneWith :: MultiMap v
82+ -> ((FilePath,v) -> IO Bool)
83+ -> IO (MultiMap v)
84+pruneWith (MultiMap mm) action = MultiMap <$> mm'
7785 where
78− go [] acc = return acc
79− go ((k,s):kss) acc = do
80− mt <- prune action s
81− case mt of
82− Nothing -> go kss acc
83− Just t -> go kss ((k,t) : acc)
86+ !mm' = I.fromAscList <$> go (I.toDescList mm) []
87+ go [] !acc = return acc
88+ go ((h,s):kss) !acc = do
89+ rs <- prune action s
90+ case rs of
91+ [] -> go kss acc
92+ _ -> go kss ((h,rs) : acc)
8493
8594----------------------------------------------------------------
8695
8796-- O(n + m) where N is the size of the second argument
88−merge :: MMap v -> MMap v -> MMap v
89−merge m1 m2 = I.unionWith (<>) m1 m2
97+merge :: MultiMap v -> MultiMap v -> MultiMap v
98+merge (MultiMap m1) (MultiMap m2) = MultiMap mm
99+ where
100+ !mm = I.unionWith (<>) m1 m2
90101
91102----------------------------------------------------------------
92103
93−prune :: (a -> IO Bool) -> NonEmpty a -> IO (Maybe (NonEmpty a))
94−prune act nxs = NE.nonEmpty <$> go (NE.toList nxs)
104+prune :: ((FilePath,v) -> IO Bool) -> [(FilePath,v)] -> IO [(FilePath,v)]
105+prune action xs0 = go xs0
95106 where
96107 go [] = return []
97108 go (x:xs) = do
98− keep <- act x
109+ keep <- action x
99110 rs <- go xs
100111 return $ if keep then x:rs else rs