on master help sign in
Make CPP work better with hie-core (#1903)

* Pull the CPP into a separate module * Pass Nothing to indicate that a text buffer shoud just be used from disk * Add save handlers, since the version changing to ModTime may have an impact * Rename contents to mbContents in one place * Change runCpp to take a Maybe StringBuffer and attempt to reuse the existing file, if it can * Add a Bazel alias for hie-core * Add notes about the sad path * Avoid one use of filePathToUri * Avoid another use of filePathToUri which went wrong for CPP output * Normalize Uri's by replacing adjacent // with a single / * Improve how CPP works if you have a modified buffer * Move textToStringBuffer out to Util * Switch to hPutStringBuffer which is in GHC 8.8 * Note why we are escaping to / * Refactoring suggested by review

commitf70eece4c7946663c8fa5d918c473a56f3ae5fb6 authorNeil Mitchell <35463327+neil-da@users.noreply.github.com> dateThu 27 Jun 2019, 23:17 (7 years ago) parent72593a2
9 files changed+99−33
src/Development/IDE/Core/Compile.hs+45−10 src/Development/IDE/Core/FileStore.hs+5−12 src/Development/IDE/Core/Shake.hs+3−3 src/Development/IDE/GHC/Compat.hs+11−1 src/Development/IDE/GHC/Error.hs+2−1 src/Development/IDE/GHC/Util.hs+9−1 src/Development/IDE/LSP/LanguageServer.hs+5−3 src/Development/IDE/LSP/Notifications.hs+6−1 src/Development/IDE/Types/Location.hs+13−1
src/Development/IDE/Core/Compile.hs+45−10
@@ -58,6 +58,7 @@ import Development.IDE.Spans.Type
5858import System.FilePath
5959import System.Directory
6060import System.IO.Extra
61+import Data.Char
6162
6263
6364-- | Contains the typechecked module and the OrigNameCache entry for
@@ -91,7 +92,7 @@ parseModule
9192 :: IdeOptions
9293 -> HscEnv
9394 -> FilePath
94− -> SB.StringBuffer
95+ -> Maybe SB.StringBuffer
9596 -> IO ([FileDiagnostic], Maybe ParsedModule)
9697parseModule IdeOptions{..} env file =
9798 fmap (either (, Nothing) (second Just)) .
@@ -310,15 +311,55 @@ getModSummaryFromBuffer fp contents dflags parsed = do
310311 , ms_parsed_mod = Nothing
311312 }
312313
314+-- | Run CPP on a file
315+runCpp :: DynFlags -> FilePath -> Maybe SB.StringBuffer -> IO SB.StringBuffer
316+runCpp dflags filename contents = withTempDir $ \dir -> do
317+ let out = dir </> takeFileName filename <.> "out"
318+ case contents of
319+ Nothing -> do
320+ -- Happy case, file is not modified, so run CPP on it in-place
321+ -- which also makes things like relative #include files work
322+ -- and means location information is correct
323+ doCpp dflags True filename out
324+ liftIO $ SB.hGetStringBuffer out
325+
326+ Just contents -> do
327+ -- Sad path, we have to create a version of the path in a temp dir
328+ -- __FILE__ macro is wrong, ignoring that for now (likely not a real issue)
329+
330+ -- Relative includes aren't going to work, so we fix that by adding to the include path.
331+ let addSelf (IncludeSpecs quote global) = IncludeSpecs (takeDirectory filename : quote) global
332+ dflags <- return dflags{includePaths = addSelf $ includePaths dflags}
333+
334+ -- Location information is wrong, so we fix that by patching it afterwards.
335+ let inp = dir </> "___HIE_CORE_MAGIC___"
336+ withBinaryFile inp WriteMode $ \h ->
337+ hPutStringBuffer h contents
338+ doCpp dflags True inp out
339+
340+ -- Fix up the filename in lines like:
341+ -- # 1 "C:/Temp/extra-dir-914611385186/___HIE_CORE_MAGIC___"
342+ let tweak x
343+ | Just x <- stripPrefix "# " x
344+ , "___HIE_CORE_MAGIC___" `isInfixOf` x
345+ , let num = takeWhile (not . isSpace) x
346+ -- important to use /, and never \ for paths, even on Windows, since then C escapes them
347+ -- and GHC gets all confused
348+ = "# " <> num <> " \"" <> map (\x -> if isPathSeparator x then '/' else x) filename <> "\""
349+ | otherwise = x
350+ stringToStringBuffer . unlines . map tweak . lines <$> readFileUTF8' out
351+
352+
313353-- | Given a buffer, flags, file path and module summary, produce a
314354-- parsed module (or errors) and any parse warnings.
315355parseFileContents
316356 :: GhcMonad m
317357 => (GHC.ParsedSource -> ([(GHC.SrcSpan, String)], GHC.ParsedSource))
318358 -> FilePath -- ^ the filename (for source locations)
319− -> SB.StringBuffer -- ^ Haskell module source text (full Unicode is supported)
359+ -> Maybe SB.StringBuffer -- ^ Haskell module source text (full Unicode is supported)
320360 -> ExceptT [FileDiagnostic] m ([FileDiagnostic], ParsedModule)
321−parseFileContents preprocessor filename contents = do
361+parseFileContents preprocessor filename mbContents = do
362+ contents <- liftIO $ maybe (hGetStringBuffer filename) return mbContents
322363 let loc = mkRealSrcLoc (mkFastString filename) 1 1
323364 dflags <- ExceptT $ parsePragmasIntoDynFlags filename contents
324365
@@ -326,13 +367,7 @@ parseFileContents preprocessor filename contents = do
326367 if not $ xopt LangExt.Cpp dflags then
327368 return (contents, dflags)
328369 else do
329− contents <- liftIO $ withTempDir $ \dir -> do
330− let inp = dir </> takeFileName filename
331− let out = dir </> takeFileName filename <.> "out"
332− let f x = if SB.atEnd x then Nothing else Just $ SB.nextChar x
333− liftIO $ writeFileUTF8 inp (unfoldr f contents)
334− doCpp dflags True inp out
335− liftIO $ SB.hGetStringBuffer out
370+ contents <- liftIO $ runCpp dflags filename mbContents
336371 dflags <- ExceptT $ parsePragmasIntoDynFlags filename contents
337372 return (contents, dflags)
338373
src/Development/IDE/Core/FileStore.hs+5−12
@@ -15,6 +15,7 @@ module Development.IDE.Core.FileStore(
1515
1616import StringBuffer
1717import Development.IDE.GHC.Orphans()
18+import Development.IDE.GHC.Util
1819
1920import Control.Concurrent.Extra
2021import qualified Data.Map.Strict as Map
@@ -31,7 +32,6 @@ import GHC.Generics
3132import Data.Either.Extra
3233import System.IO.Error
3334import qualified Data.ByteString.Char8 as BS
34−import qualified StringBuffer as SB
3535import Development.IDE.Types.Diagnostics
3636import Development.IDE.Types.Location
3737import qualified Data.Rope.UTF16 as Rope
@@ -74,8 +74,8 @@ makeLSPVFSHandle lspFuncs = VFSHandle
7474 }
7575
7676
77−-- | Get the contents of a file, either dirty (if the buffer is modified) or from disk.
78−type instance RuleResult GetFileContents = (FileVersion, StringBuffer)
77+-- | Get the contents of a file, either dirty (if the buffer is modified) or Nothing to mean use from disk.
78+type instance RuleResult GetFileContents = (FileVersion, Maybe StringBuffer)
7979
8080-- | Does the file exist.
8181type instance RuleResult GetFileExists = Bool
@@ -128,9 +128,7 @@ getFileContentsRule vfs =
128128 time <- use_ GetModificationTime file
129129 res <- liftIO $ ideTryIOException file $ do
130130 mbVirtual <- getVirtualFile vfs $ filePathToUri' file
131− case mbVirtual of
132− Just (VirtualFile _ rope _) -> return $ textToStringBuffer $ Rope.toText rope
133− Nothing -> hGetStringBuffer (fromNormalizedFilePath file)
131+ pure $ textToStringBuffer . Rope.toText . _text <$> mbVirtual
134132 case res of
135133 Left err -> return ([err], Nothing)
136134 Right contents -> return ([], Just (time, contents))
@@ -142,7 +140,7 @@ ideTryIOException fp act =
142140 <$> try act
143141
144142
145−getFileContents :: NormalizedFilePath -> Action (FileVersion, StringBuffer)
143+getFileContents :: NormalizedFilePath -> Action (FileVersion, Maybe StringBuffer)
146144getFileContents = use_ GetFileContents
147145
148146getFileExists :: NormalizedFilePath -> Action Bool
@@ -180,8 +178,3 @@ setSomethingModified state = do
180178 when (isJust setVirtualFileContents) $
181179 fail "setSomethingModified can't be called on this type of VFSHandle"
182180 void $ shakeRun state [] (const $ pure ())
183−
184−
185−-- would be nice to do this more efficiently...
186−textToStringBuffer :: T.Text -> SB.StringBuffer
187−textToStringBuffer = SB.stringToStringBuffer . T.unpack
src/Development/IDE/Core/Shake.hs+3−3
@@ -400,13 +400,13 @@ updateFileDiagnostics fp k ShakeExtras{diagnostics, state} current = do
400400 let newDiags = getFileDiagnostics fp newDiagsStore
401401 pure (newDiagsStore, (newDiags, oldDiags))
402402 when (newDiags /= oldDiags) $
403− sendEvent $ publishDiagnosticsNotification (fromNormalizedFilePath fp) newDiags
403+ sendEvent $ publishDiagnosticsNotification fp newDiags
404404
405−publishDiagnosticsNotification :: FilePath -> [Diagnostic] -> LSP.FromServerMessage
405+publishDiagnosticsNotification :: NormalizedFilePath -> [Diagnostic] -> LSP.FromServerMessage
406406publishDiagnosticsNotification fp diags =
407407 LSP.NotPublishDiagnostics $
408408 LSP.NotificationMessage "2.0" LSP.TextDocumentPublishDiagnostics $
409− LSP.PublishDiagnosticsParams (LSP.filePathToUri fp) (List diags)
409+ LSP.PublishDiagnosticsParams (fromNormalizedUri $ filePathToUri' fp) (List diags)
410410
411411setPriority :: (Enum a) => a -> Action ()
412412setPriority p =
src/Development/IDE/GHC/Compat.hs+11−1
@@ -9,9 +9,12 @@ module Development.IDE.GHC.Compat(
99 HieFile(..),
1010 mkHieFile,
1111 writeHieFile,
12− readHieFile
12+ readHieFile,
13+ hPutStringBuffer
1314 ) where
1415
16+import StringBuffer
17+
1518#ifndef GHC_STABLE
1619import HieAst
1720import HieBin
@@ -22,7 +25,14 @@ import GhcPlugins
2225import NameCache
2326import Avail
2427import TcRnTypes
28+import System.IO
29+import Foreign.ForeignPtr
30+
2531
32+hPutStringBuffer :: Handle -> StringBuffer -> IO ()
33+hPutStringBuffer hdl (StringBuffer buf len cur)
34+ = withForeignPtr (plusForeignPtr buf cur) $ \ptr ->
35+ hPutBuf hdl ptr len
2636
2737mkHieFile :: ModSummary -> TcGblEnv -> RenamedSource -> Hsc HieFile
2838mkHieFile _ _ _ = return (HieFile () [])
src/Development/IDE/GHC/Error.hs+2−1
@@ -70,7 +70,8 @@ srcSpanToFilename (RealSrcSpan real) = FS.unpackFS $ srcSpanFile real
7070
7171srcSpanToLocation :: SrcSpan -> Location
7272srcSpanToLocation src =
73− Location (filePathToUri $ srcSpanToFilename src) (srcSpanToRange src)
73+ -- important that the URI's we produce have been properly normalized, otherwise they point at weird places in VS Code
74+ Location (fromNormalizedUri $ filePathToUri' $ toNormalizedFilePath $ srcSpanToFilename src) (srcSpanToRange src)
7475
7576-- | Convert a GHC severity to a DAML compiler Severity. Severities below
7677-- "Warning" level are dropped (returning Nothing).
src/Development/IDE/GHC/Util.hs+9−1
@@ -13,7 +13,8 @@ module Development.IDE.GHC.Util(
1313 modifyDynFlags,
1414 fakeDynFlags,
1515 prettyPrint,
16− runGhcEnv
16+ runGhcEnv,
17+ textToStringBuffer
1718 ) where
1819
1920import Config
@@ -25,6 +26,9 @@ import Data.IORef
2526import Control.Exception
2627import FileCleanup
2728import Platform
29+import qualified Data.Text as T
30+import StringBuffer
31+
2832
2933----------------------------------------------------------------------
3034-- GHC setup
@@ -47,6 +51,10 @@ lookupPackageConfig unitId env =
4751 getPackageConfigMap $ hsc_dflags env
4852
4953
54+-- would be nice to do this more efficiently...
55+textToStringBuffer :: T.Text -> StringBuffer
56+textToStringBuffer = stringToStringBuffer . T.unpack
57+
5058
5159prettyPrint :: Outputable a => a -> String
5260prettyPrint = showSDoc fakeDynFlags . ppr
src/Development/IDE/LSP/LanguageServer.hs+5−3
@@ -114,6 +114,8 @@ data Message
114114
115115
116116modifyOptions :: LSP.Options -> LSP.Options
117−modifyOptions x = x{LSP.textDocumentSync = Just orig{_openClose=Just True, _change=Just TdSyncIncremental}}
118− where orig = fromMaybe tdsDefault $ LSP.textDocumentSync x
119− tdsDefault = TextDocumentSyncOptions Nothing Nothing Nothing Nothing Nothing
117+modifyOptions x = x{LSP.textDocumentSync = Just $ tweak orig}
118+ where
119+ tweak x = x{_openClose=Just True, _change=Just TdSyncIncremental, _save=Just $ SaveOptions Nothing}
120+ orig = fromMaybe tdsDefault $ LSP.textDocumentSync x
121+ tdsDefault = TextDocumentSyncOptions Nothing Nothing Nothing Nothing Nothing
src/Development/IDE/LSP/Notifications.hs+6−1
@@ -43,10 +43,15 @@ setHandlersNotifications = PartialHandlers $ \WithMessage{..} x -> return x
4343 setSomethingModified ide
4444 logInfo (ideLogger ide) $ "Modified text document: " <> getUri _uri
4545
46+ ,LSP.didSaveTextDocumentNotificationHandler = withNotification (LSP.didSaveTextDocumentNotificationHandler x) $
47+ \ide (DidSaveTextDocumentParams TextDocumentIdentifier{_uri}) -> do
48+ setSomethingModified ide
49+ logInfo (ideLogger ide) $ "Saved text document: " <> getUri _uri
50+
4651 ,LSP.didCloseTextDocumentNotificationHandler = withNotification (LSP.didCloseTextDocumentNotificationHandler x) $
4752 \ide (DidCloseTextDocumentParams TextDocumentIdentifier{_uri}) -> do
4853 setSomethingModified ide
4954 whenUriFile ide _uri $ \file ->
5055 modifyFilesOfInterest ide (S.delete file)
5156 logInfo (ideLogger ide) $ "Closed text document: " <> getUri _uri
52− }
57+ }
src/Development/IDE/Types/Location.hs+13−1
@@ -30,6 +30,7 @@ import Data.Maybe as Maybe
3030import Data.Hashable
3131import Data.String
3232import System.FilePath
33+import System.Info.Extra
3334import qualified Language.Haskell.LSP.Types as LSP
3435import Language.Haskell.LSP.Types as LSP (
3536 filePathToUri
@@ -55,7 +56,18 @@ toNormalizedFilePath fp = NormalizedFilePath $ normalise' fp
5556 -- also normalises things like the case of the drive letter
5657 -- which NormalizedUri does not normalise so we get VFS lookup failures.
5758 normalise' :: FilePath -> FilePath
58− normalise' = map (\c -> if isPathSeparator c then pathSeparator else c)
59+ normalise' = oneSlash . map (\c -> if isPathSeparator c then pathSeparator else c)
60+
61+ -- Allow double slashes as the very first element of the path for UNC drives on Windows
62+ -- otherwise turn adjacent slashes into one. These slashes often arise from dodgy CPP
63+ oneSlash :: FilePath -> FilePath
64+ oneSlash (x:xs) | isWindows = x : f xs
65+ oneSlash xs = f xs
66+
67+ f (x:y:xs) | isPathSeparator x, isPathSeparator y = f (x:xs)
68+ f (x:xs) = x : f xs
69+ f [] = []
70+
5971
6072fromNormalizedFilePath :: NormalizedFilePath -> FilePath
6173fromNormalizedFilePath (NormalizedFilePath fp) = fp