🔥 text-icu.

semantic-diff.cabal

@@ -119,7 +119,6 @@ library
                      , split
                      , template-haskell
                      , text >= 1.2.1.3
-                     , text-icu
                      , these
                      , haskell-tree-sitter
                      , vector
@@ -196,7 +195,6 @@ test-suite test
                      , regex-compat
                      , semantic-diff
                      , text >= 1.2.1.3
-                     , text-icu
                      , unordered-containers
                      , these
                      , vector

src/Command/Files.hs

@@ -3,7 +3,6 @@ module Command.Files
 ( readFile
 , readBlobPairsFromHandle
 , readBlobsFromHandle
-, transcode
 , languageForFilePath
 ) where
 
@@ -16,8 +15,6 @@ import Language
 import Prologue hiding (readFile)
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as BL
-import qualified Data.Text.ICU.Convert as Convert
-import qualified Data.Text.ICU.Detect as Detect
 import Prelude (fail)
 import Source hiding (path)
 import System.FilePath
@@ -27,15 +24,7 @@ import System.FilePath
 readFile :: FilePath -> Maybe Language -> IO SourceBlob
 readFile path language = do
   raw <- (Just <$> B.readFile path) `catch` (const (pure Nothing) :: IOException -> IO (Maybe ByteString))
-  source <- traverse transcode raw
-  pure $ fromMaybe (emptySourceBlob path) (sourceBlob path language <$> source)
-
--- | Transcode a ByteString to a unicode Source.
-transcode :: B.ByteString -> IO Source
-transcode text = fromText <$> do
-  match <- Detect.detectCharset text
-  converter <- Convert.open match Nothing
-  pure $ Convert.toUnicode converter text
+  pure $ fromMaybe (emptySourceBlob path) (sourceBlob path language . Source <$> raw)
 
 -- | Return a language based on a FilePath's extension, or Nothing if extension is not found or not supported.
 languageForFilePath :: FilePath -> Maybe Language

src/Command/Git.hs

@@ -61,9 +61,8 @@ blobForPathInTree tree path language = do
     Just (BlobEntry entryOid entryKind) -> do
       blob <- reportGitmon "cat-file" $ lookupBlob entryOid
       contents <- blobToByteString blob
-      transcoded <- liftIO $ transcode contents
       let oid = renderObjOid $ blobOid blob
-      pure (SourceBlob transcoded (toS oid) path (Just (toSourceKind entryKind)) language)
+      pure (SourceBlob (Source contents) (toS oid) path (Just (toSourceKind entryKind)) language)
     _ -> pure (emptySourceBlob path)
   where
     toSourceKind :: Git.BlobKind -> SourceKind

src/Semantic.hs

@@ -57,6 +57,7 @@ parseBlob renderer blob@SourceBlob{..} = case (renderer, blobLanguage) of
   where syntaxParser = parserForLanguage blobLanguage
 
 
+
 diffBlobPairs :: (Monoid output, StringConv output ByteString) => DiffRenderer output -> [Both SourceBlob] -> Task ByteString
 diffBlobPairs renderer = fmap toS . distributeFoldMap (diffBlobPair renderer) . filter (any blobExists)