on master help sign in
Speed up file modification checks (#2317)

Speed up file modification checks Summary: `getModificationTime` from the `directory` package is really slow. The `unix` package is faster but still slow. This PR brings the time spent checking file modifications (which is required on every change) from ~0.5s to ~0.15s.

commit01d84a6057d5e11350fa56d4a3d5cddb304591da authorMoritz Kiefer <moritz.kiefer@purelyfunctional.org> dateMon 29 Jul 2019, 16:19 (7 years ago) parent3e16335
4 files changed+78−11
BUILD.bazel+13−1 cbits/getmodtime.c+21 src/Development/IDE/Core/FileStore.hs+40−9 src/Development/IDE/Core/Shake.hs+4−1
BUILD.bazel+13−1
@@ -39,7 +39,7 @@ depends = [
3939 "transformers",
4040 "unordered-containers",
4141 "utf8-string",
42−]
42+] + ([] if is_windows else ["unix"])
4343
4444hidden = [
4545 "Development.IDE.Core.Compile",
@@ -69,8 +69,19 @@ da_haskell_library(
6969 hidden_modules = hidden,
7070 src_strip_prefix = "src",
7171 visibility = ["//visibility:public"],
72+ deps = [] if is_windows else [":getmodtime"],
7273)
7374
75+# Used in getModificationTimeRule in Development.IDE.Core.FileStore
76+cc_library(
77+ name = "getmodtime",
78+ srcs = glob(["cbits/getmodtime.c"]),
79+ copts = [
80+ "-Wall",
81+ "-Werror",
82+ ],
83+) if not is_windows else None
84+
7485da_haskell_library(
7586 name = "hie-core-public",
7687 srcs = glob(["src/**/*.hs"]),
@@ -89,6 +100,7 @@ da_haskell_library(
89100 ],
90101 src_strip_prefix = "src",
91102 visibility = ["//visibility:public"],
103+ deps = [] if is_windows else [":getmodtime"],
92104)
93105
94106da_haskell_binary(
cbits/getmodtime.cnew file+21
@@ -0,0 +1,21 @@
1+// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
2+// SPDX-License-Identifier: Apache-2.0
3+
4+#include <sys/stat.h>
5+#include <time.h>
6+int getmodtime(const char* pathname, time_t* sec, long* nsec) {
7+ struct stat s;
8+ int r = stat(pathname, &s);
9+ if (r != 0) {
10+ return r;
11+ }
12+#ifdef __APPLE__
13+ *sec = s.st_mtimespec.tv_sec;
14+ *nsec = s.st_mtimespec.tv_nsec;
15+#else
16+ *sec = s.st_mtim.tv_sec;
17+ *nsec = s.st_mtim.tv_nsec;
18+#endif
19+ return 0;
20+}
21+
src/Development/IDE/Core/FileStore.hs+40−9
@@ -1,6 +1,6 @@
11-- Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
22-- SPDX-License-Identifier: Apache-2.0
3−
3+{-# LANGUAGE CPP #-}
44{-# LANGUAGE TypeFamilies #-}
55
66module Development.IDE.Core.FileStore(
@@ -21,7 +21,6 @@ import Control.Concurrent.Extra
2121import qualified Data.Map.Strict as Map
2222import Data.Maybe
2323import qualified Data.Text as T
24−import Data.Time.Clock
2524import Control.Monad.Extra
2625import qualified System.Directory as Dir
2726import Development.Shake
@@ -35,7 +34,17 @@ import qualified Data.ByteString.Char8 as BS
3534import Development.IDE.Types.Diagnostics
3635import Development.IDE.Types.Location
3736import qualified Data.Rope.UTF16 as Rope
38−import Data.Time
37+
38+#ifdef mingw32_HOST_OS
39+import Data.Time
40+#else
41+import Foreign.C.String
42+import Foreign.C.Types
43+import Foreign.Marshal (alloca)
44+import Foreign.Ptr
45+import Foreign.Storable
46+import qualified System.Posix.Error as Posix
47+#endif
3948
4049import Language.Haskell.LSP.Core
4150import Language.Haskell.LSP.VFS
@@ -102,24 +111,46 @@ getFileExistsRule vfs =
102111 return (Just $ if res then BS.singleton '1' else BS.empty, ([], Just res))
103112
104113
105−showTimePrecise :: UTCTime -> String
106−showTimePrecise UTCTime{..} = show (toModifiedJulianDay utctDay, diffTimeToPicoseconds utctDayTime)
107−
108114getModificationTimeRule :: VFSHandle -> Rules ()
109115getModificationTimeRule vfs =
110116 defineEarlyCutoff $ \GetModificationTime file -> do
111117 let file' = fromNormalizedFilePath file
112− let wrap time = (Just $ BS.pack $ showTimePrecise time, ([], Just $ ModificationTime time))
118+ let wrap time = (Just time, ([], Just $ ModificationTime time))
113119 alwaysRerun
114120 mbVirtual <- liftIO $ getVirtualFile vfs $ filePathToUri' file
115121 case mbVirtual of
116122 Just (VirtualFile ver _ _) -> pure (Just $ BS.pack $ show ver, ([], Just $ VFSVersion ver))
117− Nothing -> liftIO $ fmap wrap (Dir.getModificationTime file')
123+ Nothing -> liftIO $ fmap wrap (getModTime file')
118124 `catch` \(e :: IOException) -> do
119125 let err | isDoesNotExistError e = "File does not exist: " ++ file'
120126 | otherwise = "IO error while reading " ++ file' ++ ", " ++ displayException e
121127 return (Nothing, ([ideErrorText file $ T.pack err], Nothing))
122−
128+ where
129+ -- Dir.getModificationTime is surprisingly slow since it performs
130+ -- a ton of conversions. Since we do not actually care about
131+ -- the format of the time, we can get away with something cheaper.
132+ -- For now, we only try to do this on Unix systems where it seems to get the
133+ -- time spent checking file modifications (which happens on every change)
134+ -- from > 0.5s to ~0.15s.
135+ -- We might also want to try speeding this up on Windows at some point.
136+ getModTime :: FilePath -> IO BS.ByteString
137+ getModTime f =
138+#ifdef mingw32_HOST_OS
139+ do time <- Dir.getModificationTime f
140+ pure $! BS.pack $ show (toModifiedJulianDay $ utctDay time, diffTimeToPicoseconds $ utctDayTime time)
141+#else
142+ withCString f $ \f' ->
143+ alloca $ \secPtr ->
144+ alloca $ \nsecPtr -> do
145+ Posix.throwErrnoPathIfMinus1Retry_ "getmodtime" f $ c_getModTime f' secPtr nsecPtr
146+ sec <- peek secPtr
147+ nsec <- peek nsecPtr
148+ pure $! BS.pack $ show sec <> "." <> show nsec
149+
150+-- Sadly even unix’s getFileStatus + modificationTimeHiRes is still about twice as slow
151+-- as doing the FFI call ourselves :(.
152+foreign import ccall "getmodtime" c_getModTime :: CString -> Ptr CTime -> Ptr CLong -> IO Int
153+#endif
123154
124155getFileContentsRule :: VFSHandle -> Rules ()
125156getFileContentsRule vfs =
src/Development/IDE/Core/Shake.hs+4−1
@@ -582,7 +582,10 @@ instance NFData GetModificationTime
582582-- | Get the modification time of a file.
583583type instance RuleResult GetModificationTime = FileVersion
584584
585−data FileVersion = VFSVersion Int | ModificationTime UTCTime
585+-- | We store the modification time as a ByteString since we need
586+-- a ByteString anyway for Shake and we do not care about how times
587+-- are represented.
588+data FileVersion = VFSVersion Int | ModificationTime BS.ByteString
586589 deriving (Show, Generic)
587590
588591instance NFData FileVersion