d19a347 · typeable/pandoc

git.iko.soy/typeable/pandoc

UTF8: Better handling of bare CRs in input files.

Previously we just stripped them out; now we convert other line ending styles to LF line endings. Closes #2132.

commitd19a347fd5e93802e80e5bd8e13f29de74a53f3c authorJohn MacFarlane <jgm@berkeley.edu> dateTue 5 May 2015, 12:41 (11 years ago) parentdb4d69a

1 file changed+8−2

unifiedsplit

src/Text/Pandoc/UTF8.hs+8−2

@@ -93,10 +93,16 @@ dropBOM :: String -> String

9393dropBOM ('\xFEFF':xs) = xs

9494dropBOM xs = xs

9595

96+filterCRs :: String -> String

97+filterCRs ('\r':'\n':xs) = '\n': filterCRs xs

98+filterCRs ('\r':xs) = '\n' : filterCRs xs

99+filterCRs (x:xs) = x : filterCRs xs

100+filterCRs [] = []

101+

96102-- | Convert UTF8-encoded ByteString to String, also

97103-- removing '\r' characters.

98104toString :: B.ByteString -> String

99−toString = filter (/='\r') . dropBOM . T.unpack . T.decodeUtf8

105+toString = filterCRs . dropBOM . T.unpack . T.decodeUtf8

100106

101107fromString :: String -> B.ByteString

102108fromString = T.encodeUtf8 . T.pack

@@ -104,7 +110,7 @@ fromString = T.encodeUtf8 . T.pack

104110-- | Convert UTF8-encoded ByteString to String, also

105111-- removing '\r' characters.

106112toStringLazy :: BL.ByteString -> String

107−toStringLazy = filter (/='\r') . dropBOM . TL.unpack . TL.decodeUtf8

113+toStringLazy = filterCRs . dropBOM . TL.unpack . TL.decodeUtf8

108114

109115fromStringLazy :: String -> BL.ByteString

110116fromStringLazy = TL.encodeUtf8 . TL.pack