25f76d5 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Add new encoding/decoding routines.

25f76d5pranaysashank6 years ago

Changelog.md+18−3

compare with master

unifiedsplit

@@ -1,5 +1,10 @@

1## Unreleased

3+### Enhancements

5+* New encoding/decoding routines, `encodeUtf8'`, `encodeLatin1'`, `decodeUtf8'`,

6+ are added, these routines fail when they encounter any invalid characters.

3### Breaking changes

8### Breaking changes

5* Deprecate `Streamly.Memory.Array` in favor of `Streamly.Data.Array.Storable.Foreign`

10* Deprecate `Streamly.Memory.Array` in favor of `Streamly.Data.Array.Storable.Foreign`

@@ -14,6 +19,12 @@

14 constraint.

19 constraint.

15* Change the associativity of combinators `serial`, `wSerial`,

20* Change the associativity of combinators `serial`, `wSerial`,

16 `ahead`, `async`, `wAsync`, `parallel` to be the same as `<>`.

21 `ahead`, `async`, `wAsync`, `parallel` to be the same as `<>`.

22+* `encodeUtf8`, `decodeUtf8` now replace any invalid character encountered

23+ during encoding/decoding with the Unicode replacement character. Use

24+ `encodeUtf8'`, `decodeUtf8'` to recover the previous functionality.

25+* `encodeLatin1` now silently truncates any character beyond 255 to incorrect

26+ characters in the input stream. Use `encodeLatin1'` to recover previous

27+ functionality.

17* Drop support for GHC 7.10.3.

28* Drop support for GHC 7.10.3.

19### Bug Fixes

30### Bug Fixes

@@ -33,9 +44,13 @@

33* The `Streamly` module is now deprecated, its functionality is subsumed

44* The `Streamly` module is now deprecated, its functionality is subsumed

34 by `Streamly.Prelude`.

45 by `Streamly.Prelude`.

35* Some functions from `Streamly` module have been renamed in `Streamly.Prelude` module:

46* Some functions from `Streamly` module have been renamed in `Streamly.Prelude` module:

36− * `foldWith` has been replaced by `concatFoldableWith`

47+ * `foldWith` to `concatFoldableWith`

37− * `foldMapWith` has been replaced by `concatMapFoldableWith`

48+ * `foldMapWith` to `concatMapFoldableWith`

38− * `forEachWith` has been replaced by `concatForFoldableWith`

49+ * `forEachWith` to `concatForFoldableWith`

50+* The following encoding/decoding routines have been renamed:

51+ * `encodeUtf8Lax` to `encodeUtf8`

52+ * `encodeLatin1Lax` to `encodeLatin1`

53+ * `decodeUtf8Lenient` to `decodeUtf8`

40## 0.7.2

55## 0.7.2

benchmark/Streamly/Benchmark/FileSystem/Handle/Read.hs+14−14

@@ -120,12 +120,12 @@ inspect $ hasNoTypeClasses 'toChunksCountBytes

120inspect $ 'toChunksCountBytes `hasNoType` ''Step

121#endif

122

123−toChunksDecodeUtf8ArraysLenient :: Handle -> IO ()

123+toChunksDecodeUtf8Arrays :: Handle -> IO ()

124−toChunksDecodeUtf8ArraysLenient =

124+toChunksDecodeUtf8Arrays =

125− S.drain . IUS.decodeUtf8ArraysLenient . IFH.toChunks

125+ S.drain . IUS.decodeUtf8Arrays . IFH.toChunks

126

127#ifdef INSPECTION

128−inspect $ hasNoTypeClasses 'toChunksDecodeUtf8ArraysLenient

128+inspect $ hasNoTypeClasses 'toChunksDecodeUtf8Arrays

129-- inspect $ 'toChunksDecodeUtf8ArraysLenient `hasNoType` ''Step

130#endif

131

@@ -146,8 +146,8 @@ o_1_space_read_chunked env =

146 toChunksSplitOn inH

147 , mkBench "countBytes" env $ \inH _ ->

148 toChunksCountBytes inH

149− , mkBenchSmall "US.decodeUtf8ArraysLenient" env $ \inH _ ->

149+ , mkBenchSmall "US.decodeUtf8Arrays" env $ \inH _ ->

150− toChunksDecodeUtf8ArraysLenient inH

150+ toChunksDecodeUtf8Arrays inH

151 ]

152 ]

153

@@ -240,14 +240,14 @@ readDecodeLatin1 inh =

240 $ SS.decodeLatin1

241 $ S.unfold FH.read inh

242

243−readDecodeUtf8Lax :: Handle -> IO ()

243+readDecodeUtf8 :: Handle -> IO ()

244−readDecodeUtf8Lax inh =

244+readDecodeUtf8 inh =

245 S.drain

246− $ SS.decodeUtf8Lax

246+ $ SS.decodeUtf8

247 $ S.unfold FH.read inh

248

249#ifdef INSPECTION

250−inspect $ hasNoTypeClasses 'readDecodeUtf8Lax

250+inspect $ hasNoTypeClasses 'readDecodeUtf8

251-- inspect $ 'readDecodeUtf8Lax `hasNoType` ''Step

252#endif

253

@@ -273,8 +273,8 @@ o_1_space_reduce_read env =

273 readCountWords inh

274

275 -- read with utf8 decoding

276− , mkBenchSmall "SS.decodeUtf8Lax" env $ \inh _ ->

276+ , mkBenchSmall "SS.decodeUtf8" env $ \inh _ ->

277− readDecodeUtf8Lax inh

277+ readDecodeUtf8 inh

278 ]

279 ]

280

@@ -505,14 +505,14 @@ o_1_space_reduce_read_split env =

505splitOnSeqUtf8 :: String -> Handle -> IO Int

506splitOnSeqUtf8 str inh =

507 (S.length $ IP.splitOnSeq (A.fromList str) FL.drain

508− $ IUS.decodeUtf8ArraysLenient

508+ $ IUS.decodeUtf8Arrays

509 $ IFH.toChunks inh) -- >>= print

510

511o_1_space_reduce_toChunks_split :: BenchEnv -> [Benchmark]

512o_1_space_reduce_toChunks_split env =

513 [ bgroup "reduce/toChunks"

514 [ mkBenchSmall ("S.splitOnSeqUtf8 \"abcdefgh\" FL.drain "

515− ++ ". US.decodeUtf8ArraysLenient") env $ \inh _ ->

515+ ++ ". US.decodeUtf8Arrays") env $ \inh _ ->

516 splitOnSeqUtf8 "abcdefgh" inh

517 , mkBenchSmall "S.splitOnSeqUtf8 \"abcdefghijklmnopqrstuvwxyz\" FL.drain"

518 env $ \inh _ -> splitOnSeqUtf8 "abcdefghijklmnopqrstuvwxyz" inh

benchmark/Streamly/Benchmark/FileSystem/Handle/ReadWrite.hs+21−21

@@ -71,8 +71,8 @@ inspect $ 'copyChunks `hasNoType` ''Step

71copyCodecUtf8ArraysLenient :: Handle -> Handle -> IO ()

72copyCodecUtf8ArraysLenient inh outh =

73 S.fold (FH.write outh)

74− $ SS.encodeUtf8

74+ $ SS.encodeUtf8'

75− $ IUS.decodeUtf8ArraysLenient

75+ $ IUS.decodeUtf8Arrays

76 $ IFH.toChunks inh

78#ifdef INSPECTION

@@ -194,7 +194,7 @@ inspect $ 'copyStream `hasNoType` ''Strict.Tuple3' -- FH.write/lchunksOf

194copyStreamLatin1 :: Handle -> Handle -> IO ()

195copyStreamLatin1 inh outh =

196 S.fold (FH.write outh)

197− $ SS.encodeLatin1

197+ $ SS.encodeLatin1'

198 $ SS.decodeLatin1

199 $ S.unfold FH.read inh

200

@@ -208,30 +208,30 @@ inspect $ 'copyStreamLatin1 `hasNoType` ''Strict.Tuple3' -- FH.write/lchunksOf

208#endif

209

210-- | Copy file

211−_copyStreamUtf8 :: Handle -> Handle -> IO ()

211+_copyStreamUtf8' :: Handle -> Handle -> IO ()

212−_copyStreamUtf8 inh outh =

212+_copyStreamUtf8' inh outh =

213 S.fold (FH.write outh)

214− $ SS.encodeUtf8

214+ $ SS.encodeUtf8'

215− $ SS.decodeUtf8

215+ $ SS.decodeUtf8'

216 $ S.unfold FH.read inh

217

218#ifdef INSPECTION

219−inspect $ hasNoTypeClasses '_copyStreamUtf8

219+inspect $ hasNoTypeClasses '_copyStreamUtf8'

220-- inspect $ '_copyStreamUtf8 `hasNoType` ''Step

221-- inspect $ '_copyStreamUtf8 `hasNoType` ''AT.FlattenState

222-- inspect $ '_copyStreamUtf8 `hasNoType` ''D.ConcatMapUState

223#endif

224

225-- | Copy file

226−copyStreamUtf8Lax :: Handle -> Handle -> IO ()

226+copyStreamUtf8 :: Handle -> Handle -> IO ()

227−copyStreamUtf8Lax inh outh =

227+copyStreamUtf8 inh outh =

228 S.fold (FH.write outh)

229 $ SS.encodeUtf8

230− $ SS.decodeUtf8Lax

230+ $ SS.decodeUtf8

231 $ S.unfold FH.read inh

232

233#ifdef INSPECTION

234−inspect $ hasNoTypeClasses 'copyStreamUtf8Lax

234+inspect $ hasNoTypeClasses 'copyStreamUtf8

235-- inspect $ 'copyStreamUtf8Lax `hasNoType` ''Step

236-- inspect $ 'copyStreamUtf8Lax `hasNoType` ''AT.FlattenState

237-- inspect $ 'copyStreamUtf8Lax `hasNoType` ''D.ConcatMapUState

@@ -245,14 +245,14 @@ o_1_space_copy_read env =

245 , mkBench "rawToFile" env $ \inh outh ->

246 copyStream inh outh

247 -- This needs an ascii file, as decode just errors out.

248− , mkBench "SS.encodeLatin1 . SS.decodeLatin1" env $ \inh outh ->

248+ , mkBench "SS.encodeLatin1' . SS.decodeLatin1" env $ \inh outh ->

249 copyStreamLatin1 inh outh

250#ifdef DEVBUILD

251 , mkBench "copyUtf8" env $ \inh outh ->

252− _copyStreamUtf8 inh outh

252+ _copyStreamUtf8' inh outh

253#endif

254 , mkBenchSmall "SS.encodeUtf8 . SS.decodeUtf8Lax" env $ \inh outh ->

255− copyStreamUtf8Lax inh outh

255+ copyStreamUtf8 inh outh

256 ]

257 ]

258

@@ -471,7 +471,7 @@ o_1_space_copy_toChunks_group_ungroup env =

471linesUnlinesCopy :: Handle -> Handle -> IO ()

472linesUnlinesCopy inh outh =

473 S.fold (FH.write outh)

474− $ SS.encodeLatin1

474+ $ SS.encodeLatin1'

475 $ IUS.unlines IUF.fromList

476 $ S.splitOnSuffix (== '\n') FL.toList

477 $ SS.decodeLatin1

@@ -489,7 +489,7 @@ linesUnlinesArrayWord8Copy inh outh =

489linesUnlinesArrayCharCopy :: Handle -> Handle -> IO ()

490linesUnlinesArrayCharCopy inh outh =

491 S.fold (FH.write outh)

492− $ SS.encodeLatin1

492+ $ SS.encodeLatin1'

493 $ IUA.unlines

494 $ IUA.lines

495 $ SS.decodeLatin1

@@ -507,9 +507,9 @@ inspect $ hasNoTypeClassesExcept 'linesUnlinesArrayCharCopy [''Storable]

507linesUnlinesArrayUtf8Copy :: Handle -> Handle -> IO ()

508linesUnlinesArrayUtf8Copy inh outh =

509 S.fold (FH.write outh)

510− $ SS.encodeLatin1

510+ $ SS.encodeLatin1'

511 $ IP.intercalate (A.fromList [10]) (pipe SS.decodeUtf8P A.read)

512− $ S.splitOnSuffix (== '\n') (IFL.lmap SS.encodeUtf8 A.write)

512+ $ S.splitOnSuffix (== '\n') (IFL.lmap SS.encodeUtf8' A.write)

513 $ SS.decodeLatin1

514 $ S.unfold FH.read inh

515-}

@@ -531,7 +531,7 @@ inspect $ hasNoTypeClasses 'wordsUnwordsCopyWord8

531wordsUnwordsCopy :: Handle -> Handle -> IO ()

532wordsUnwordsCopy inh outh =

533 S.fold (FH.write outh)

534− $ SS.encodeLatin1

534+ $ SS.encodeLatin1'

535 $ IUS.unwords IUF.fromList

536 -- XXX This pipeline does not fuse with wordsBy but fuses with splitOn

537 -- with -funfolding-use-threshold=300. With wordsBy it does not fuse

@@ -553,7 +553,7 @@ wordsUnwordsCopy inh outh =

553wordsUnwordsCharArrayCopy :: Handle -> Handle -> IO ()

554wordsUnwordsCharArrayCopy inh outh =

555 S.fold (FH.write outh)

556− $ SS.encodeLatin1

556+ $ SS.encodeLatin1'

557 $ IUA.unwords

558 $ IUA.words

559 $ SS.decodeLatin1

examples/FileSinkServer.hs+1−1

@@ -23,7 +23,7 @@ main = do

23 file <- fmap head getArgs

24 withFile file AppendMode

25 (\src -> S.fold (FH.write src)

26− $ encodeLatin1Lax

26+ $ encodeLatin1

27 $ S.concatUnfold A.read

28 $ S.concatMapWith S.parallel use

29 $ S.unfold TCP.acceptOnPort 8090)

examples/WordCount.hs+1−1

@@ -335,7 +335,7 @@ countCharSerial (Counts l w c wasSpace) ch =

335_wc_mwl_serial :: Handle -> IO ()

336_wc_mwl_serial src = print =<< (

337 S.foldl' countCharSerial (Counts 0 0 0 True)

338− $ S.decodeUtf8Lax

338+ $ S.decodeUtf8

339 $ S.unfold FH.read src)

340

341-------------------------------------------------------------------------------

src/Streamly/Data/Unicode/Stream.hs+7−5

@@ -33,7 +33,7 @@

33-- stream as @Array Char@ using the array 'Streamly.Data.Array.Storable.Foreign.write' fold.

34-- The 'Array' type provides a more compact representation and pinned memory

35-- reducing GC overhead. If space efficiency is a concern you can use

36−-- 'encodeUtf8' on the 'Char' stream before writing it to an 'Array' providing

36+-- 'encodeUtf8'' on the 'Char' stream before writing it to an 'Array' providing

37-- an even more compact representation.

38--

39-- = String Literals

@@ -65,18 +65,15 @@

65-- lived strings in memory.

66--

67module Streamly.Data.Unicode.Stream

68− {-# DEPRECATED "Use Streamly.Unicode.Stream instead" #-}

68+ {-# DEPRECATED "Use \"Streamly.Unicode.Stream\" instead" #-}

69 (

70 -- * Construction (Decoding)

71 decodeLatin1

72 , decodeUtf8

73− , decodeUtf8Lax

75 -- * Elimination (Encoding)

74 -- * Elimination (Encoding)

76 , encodeLatin1

75 , encodeLatin1

77− , encodeLatin1Lax

78 , encodeUtf8

76 , encodeUtf8

79− , encodeUtf8Lax

80 {-

77 {-

81 -- * Operations on character strings

78 -- * Operations on character strings

82 , strip -- (dropAround isSpace)

79 , strip -- (dropAround isSpace)

@@ -90,6 +87,11 @@ module Streamly.Data.Unicode.Stream

90 -- , words

87 -- , words

91 -- , unlines

88 -- , unlines

92 -- , unwords

89 -- , unwords

90+

91+ -- * Deprecations

92+ , decodeUtf8Lax

93+ , encodeLatin1Lax

94+ , encodeUtf8Lax

93 )

95 )

94where

96where

src/Streamly/Internal/FileSystem/Event/Darwin.hs+1−1

@@ -980,7 +980,7 @@ isLastHardLink = getFlag kFSEventStreamEventFlagItemIsLastHardlink

980-- | Convert an 'Event' record to a String representation.

981showEvent :: Event -> String

982showEvent ev@Event{..} =

983− let path = runIdentity $ S.toList $ U.decodeUtf8 $ A.toStream eventAbsPath

983+ let path = runIdentity $ S.toList $ U.decodeUtf8' $ A.toStream eventAbsPath

984 in "--------------------------"

985 ++ "\nId = " ++ show eventId

986 ++ "\nPath = " ++ show path

src/Streamly/Internal/FileSystem/Event/Linux.hs+1−1

@@ -552,7 +552,7 @@ foreign import ccall unsafe

552 :: CInt -> CString -> CUInt -> IO CInt

553

554utf8ToString :: Array Word8 -> String

555−utf8ToString = runIdentity . S.toList . U.decodeUtf8 . A.toStream

555+utf8ToString = runIdentity . S.toList . U.decodeUtf8' . A.toStream

556

557#if !MIN_VERSION_base(4,10,0)

558-- | Turn an existing Handle into a file descriptor. This function throws an

src/Streamly/Internal/Unicode/Stream.hs+168−52

@@ -16,20 +16,24 @@ module Streamly.Internal.Unicode.Stream

16 -- * Construction (Decoding)

17 decodeLatin1

18 , decodeUtf8

19− , decodeUtf8Lax

19+ , decodeUtf8'

20+ , decodeUtf8_

20 , DecodeError(..)

21 , DecodeError(..)

21 , DecodeState

22 , DecodeState

22 , CodePoint

23 , CodePoint

23 , decodeUtf8Either

24 , decodeUtf8Either

24 , resumeDecodeUtf8Either

25 , resumeDecodeUtf8Either

25 , decodeUtf8Arrays

26 , decodeUtf8Arrays

26− , decodeUtf8ArraysLenient

27+ , decodeUtf8Arrays'

28+ , decodeUtf8Arrays_

28 -- * Elimination (Encoding)

30 -- * Elimination (Encoding)

29 , encodeLatin1

31 , encodeLatin1

30− , encodeLatin1Lax

32+ , encodeLatin1'

33+ , encodeLatin1_

31 , encodeUtf8

34 , encodeUtf8

32− , encodeUtf8Lax

35+ , encodeUtf8'

36+ , encodeUtf8_

33 {-

37 {-

34 -- * Operations on character strings

38 -- * Operations on character strings

35 , strip -- (dropAround isSpace)

39 , strip -- (dropAround isSpace)

@@ -38,12 +42,16 @@ module Streamly.Internal.Unicode.Stream

39 -- * StreamD UTF8 Encoding / Decoding transformations.

43 -- * StreamD UTF8 Encoding / Decoding transformations.

40 , decodeUtf8D

44 , decodeUtf8D

45+ , decodeUtf8D'

46+ , decodeUtf8D_

41 , encodeUtf8D

47 , encodeUtf8D

42− , decodeUtf8LenientD

48+ , encodeUtf8D'

49+ , encodeUtf8D_

43 , decodeUtf8EitherD

50 , decodeUtf8EitherD

44 , resumeDecodeUtf8EitherD

51 , resumeDecodeUtf8EitherD

45 , decodeUtf8ArraysD

52 , decodeUtf8ArraysD

46− , decodeUtf8ArraysLenientD

53+ , decodeUtf8ArraysD'

54+ , decodeUtf8ArraysD_

48 -- * Transformation

56 -- * Transformation

49 , stripStart

57 , stripStart

@@ -51,12 +59,17 @@ module Streamly.Internal.Unicode.Stream

51 , words

59 , words

52 , unlines

60 , unlines

53 , unwords

61 , unwords

62+

63+ -- * Deprecations

64+ , decodeUtf8Lax

65+ , encodeLatin1Lax

66+ , encodeUtf8Lax

54 )

67 )

55where

68where

57import Control.Monad.IO.Class (MonadIO, liftIO)

70import Control.Monad.IO.Class (MonadIO, liftIO)

58import Data.Bits (shiftR, shiftL, (.|.), (.&.))

71import Data.Bits (shiftR, shiftL, (.|.), (.&.))

59−import Data.Char (ord)

72+import Data.Char (chr, ord)

60import Data.Word (Word8)

73import Data.Word (Word8)

61import Foreign.ForeignPtr (touchForeignPtr)

74import Foreign.ForeignPtr (touchForeignPtr)

62import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)

75import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)

@@ -118,6 +131,7 @@ ord4 c = assert (n >= 0x10000) (WCons x1 (WCons x2 (WCons x3 (WCons x4 WNil))))

118data CodingFailureMode

131data CodingFailureMode

119 = TransliterateCodingFailure

132 = TransliterateCodingFailure

120 | ErrorOnCodingFailure

133 | ErrorOnCodingFailure

134+ | DropOnCodingFailure

121 deriving (Show)

135 deriving (Show)

122

136

123{-# INLINE replacementChar #-}

137{-# INLINE replacementChar #-}

@@ -266,12 +280,14 @@ decodeUtf8WithD cfm (Stream step state) =

266 case cfm of

280 case cfm of

267 ErrorOnCodingFailure -> error e

281 ErrorOnCodingFailure -> error e

268 TransliterateCodingFailure -> YieldAndContinue replacementChar s

282 TransliterateCodingFailure -> YieldAndContinue replacementChar s

283+ DropOnCodingFailure -> s

269

284

270 {-# INLINE handleUnderflow #-}

285 {-# INLINE handleUnderflow #-}

271 handleUnderflow =

286 handleUnderflow =

272 case cfm of

287 case cfm of

273 ErrorOnCodingFailure -> error $ prefix ++ "Not enough input"

288 ErrorOnCodingFailure -> error $ prefix ++ "Not enough input"

274 TransliterateCodingFailure -> YieldAndContinue replacementChar Done

289 TransliterateCodingFailure -> YieldAndContinue replacementChar Done

290+ DropOnCodingFailure -> Done

275

291

276 {-# INLINE_LATE step' #-}

292 {-# INLINE_LATE step' #-}

277 step' _ gst (FreshPointDecodeInit st) = do

293 step' _ gst (FreshPointDecodeInit st) = do

@@ -337,11 +353,15 @@ decodeUtf8WithD cfm (Stream step state) =

337

353

338{-# INLINE decodeUtf8D #-}

354{-# INLINE decodeUtf8D #-}

339decodeUtf8D :: Monad m => Stream m Word8 -> Stream m Char

355decodeUtf8D :: Monad m => Stream m Word8 -> Stream m Char

340−decodeUtf8D = decodeUtf8WithD ErrorOnCodingFailure

356+decodeUtf8D = decodeUtf8WithD TransliterateCodingFailure

357+

358+{-# INLINE decodeUtf8D' #-}

359+decodeUtf8D' :: Monad m => Stream m Word8 -> Stream m Char

360+decodeUtf8D' = decodeUtf8WithD ErrorOnCodingFailure

341

361

342−{-# INLINE decodeUtf8LenientD #-}

362+{-# INLINE decodeUtf8D_ #-}

343−decodeUtf8LenientD :: Monad m => Stream m Word8 -> Stream m Char

363+decodeUtf8D_ :: Monad m => Stream m Word8 -> Stream m Char

344−decodeUtf8LenientD = decodeUtf8WithD TransliterateCodingFailure

364+decodeUtf8D_ = decodeUtf8WithD DropOnCodingFailure

345

365

346{-# INLINE_NORMAL resumeDecodeUtf8EitherD #-}

366{-# INLINE_NORMAL resumeDecodeUtf8EitherD #-}

347resumeDecodeUtf8EitherD

367resumeDecodeUtf8EitherD

@@ -455,6 +475,7 @@ decodeUtf8ArraysWithD cfm (Stream step state) =

455 case cfm of

475 case cfm of

456 ErrorOnCodingFailure -> error e

476 ErrorOnCodingFailure -> error e

457 TransliterateCodingFailure -> YAndC replacementChar s

477 TransliterateCodingFailure -> YAndC replacementChar s

478+ DropOnCodingFailure -> s

458 {-# INLINE inputUnderflow #-}

479 {-# INLINE inputUnderflow #-}

459 inputUnderflow =

480 inputUnderflow =

460 case cfm of

481 case cfm of

@@ -462,6 +483,7 @@ decodeUtf8ArraysWithD cfm (Stream step state) =

462 error

483 error

463 "Streamly.Internal.Data.Stream.StreamD.decodeUtf8ArraysWith: Input Underflow"

484 "Streamly.Internal.Data.Stream.StreamD.decodeUtf8ArraysWith: Input Underflow"

464 TransliterateCodingFailure -> YAndC replacementChar D

485 TransliterateCodingFailure -> YAndC replacementChar D

486+ DropOnCodingFailure -> D

465 {-# INLINE_LATE step' #-}

487 {-# INLINE_LATE step' #-}

466 step' _ gst (OuterLoop st Nothing) = do

488 step' _ gst (OuterLoop st Nothing) = do

467 r <- step (adaptState gst) st

489 r <- step (adaptState gst) st

@@ -541,14 +563,21 @@ decodeUtf8ArraysD ::

541 MonadIO m

563 MonadIO m

542 => Stream m (A.Array Word8)

564 => Stream m (A.Array Word8)

543 -> Stream m Char

565 -> Stream m Char

544−decodeUtf8ArraysD = decodeUtf8ArraysWithD ErrorOnCodingFailure

566+decodeUtf8ArraysD = decodeUtf8ArraysWithD TransliterateCodingFailure

567+

568+{-# INLINE decodeUtf8ArraysD' #-}

569+decodeUtf8ArraysD' ::

570+ MonadIO m

571+ => Stream m (A.Array Word8)

572+ -> Stream m Char

573+decodeUtf8ArraysD' = decodeUtf8ArraysWithD ErrorOnCodingFailure

545

574

546−{-# INLINE decodeUtf8ArraysLenientD #-}

575+{-# INLINE decodeUtf8ArraysD_ #-}

547−decodeUtf8ArraysLenientD ::

576+decodeUtf8ArraysD_ ::

548 MonadIO m

577 MonadIO m

549 => Stream m (A.Array Word8)

578 => Stream m (A.Array Word8)

550 -> Stream m Char

579 -> Stream m Char

551−decodeUtf8ArraysLenientD = decodeUtf8ArraysWithD TransliterateCodingFailure

580+decodeUtf8ArraysD_ = decodeUtf8ArraysWithD DropOnCodingFailure

552

581

553data EncodeState s = EncodeState s !WList

582data EncodeState s = EncodeState s !WList

554

583

@@ -556,9 +585,9 @@ data EncodeState s = EncodeState s !WList

556-- too much code bloat or some trouble with fusion. So keeping only two yield

585-- too much code bloat or some trouble with fusion. So keeping only two yield

557-- points for now, one for the ascii chars (fast path) and one for all other

586-- points for now, one for the ascii chars (fast path) and one for all other

558-- paths (slow path).

587-- paths (slow path).

559−{-# INLINE_NORMAL encodeUtf8D #-}

588+{-# INLINE_NORMAL encodeUtf8D' #-}

560−encodeUtf8D :: Monad m => Stream m Char -> Stream m Word8

589+encodeUtf8D' :: Monad m => Stream m Char -> Stream m Word8

561−encodeUtf8D (Stream step state) = Stream step' (EncodeState state WNil)

590+encodeUtf8D' (Stream step state) = Stream step' (EncodeState state WNil)

562 where

591 where

563 {-# INLINE_LATE step' #-}

592 {-# INLINE_LATE step' #-}

564 step' gst (EncodeState st WNil) = do

593 step' gst (EncodeState st WNil) = do

@@ -585,7 +614,9 @@ encodeUtf8D (Stream step state) = Stream step' (EncodeState state WNil)

585-- | Decode a stream of bytes to Unicode characters by mapping each byte to a

614-- | Decode a stream of bytes to Unicode characters by mapping each byte to a

586-- corresponding Unicode 'Char' in 0-255 range.

615-- corresponding Unicode 'Char' in 0-255 range.

587--

616--

588−-- /Since: 0.7.0/

617+-- /Since: 0.7.0 ("Streamly.Data.Unicode.Stream")/

618+--

619+-- @since 0.8.0

589{-# INLINE decodeLatin1 #-}

620{-# INLINE decodeLatin1 #-}

590decodeLatin1 :: (IsStream t, Monad m) => t m Word8 -> t m Char

621decodeLatin1 :: (IsStream t, Monad m) => t m Word8 -> t m Char

591decodeLatin1 = S.map (unsafeChr . fromIntegral)

622decodeLatin1 = S.map (unsafeChr . fromIntegral)

@@ -594,10 +625,10 @@ decodeLatin1 = S.map (unsafeChr . fromIntegral)

594-- to a byte in 0-255 range. Throws an error if the input stream contains

625-- to a byte in 0-255 range. Throws an error if the input stream contains

595-- characters beyond 255.

626-- characters beyond 255.

596--

627--

597−-- /Since: 0.7.0/

628+-- @since 0.8.0

598−{-# INLINE encodeLatin1 #-}

629+{-# INLINE encodeLatin1' #-}

599−encodeLatin1 :: (IsStream t, Monad m) => t m Char -> t m Word8

630+encodeLatin1' :: (IsStream t, Monad m) => t m Char -> t m Word8

600−encodeLatin1 = S.map convert

631+encodeLatin1' = S.map convert

601 where

632 where

602 convert c =

633 convert c =

603 let codepoint = ord c

634 let codepoint = ord c

@@ -606,38 +637,71 @@ encodeLatin1 = S.map convert

606 "input char codepoint " ++ show codepoint

637 "input char codepoint " ++ show codepoint

607 else fromIntegral codepoint

638 else fromIntegral codepoint

608

639

609−-- | Like 'encodeLatin1' but silently truncates and maps input characters beyond

640+-- | Like 'encodeLatin1'' but silently truncates and maps input characters beyond

610-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown

641-- 255 to (incorrect) chars in 0-255 range. No error or exception is thrown

611-- when such truncation occurs.

642-- when such truncation occurs.

612--

643--

613−-- /Since: 0.7.0/

644+-- /Since: 0.7.0 ("Streamly.Data.Unicode.Stream")/

645+--

646+-- /Since: 0.8.0 (Lenient Behaviour)/

647+{-# INLINE encodeLatin1 #-}

648+encodeLatin1 :: (IsStream t, Monad m) => t m Char -> t m Word8

649+encodeLatin1 = S.map (fromIntegral . ord)

650+

651+-- | Like 'encodeLatin1' but drops the input characters beyond 255.

652+--

653+-- @since 0.8.0

654+{-# INLINE encodeLatin1_ #-}

655+encodeLatin1_ :: (IsStream t, Monad m) => t m Char -> t m Word8

656+encodeLatin1_ = S.map (fromIntegral . ord) . S.filter (<= chr (255))

657+

658+-- | Same as 'encodeLatin1'

659+--

660+{-# DEPRECATED encodeLatin1Lax "Please use 'encodeLatin1' instead" #-}

614{-# INLINE encodeLatin1Lax #-}

661{-# INLINE encodeLatin1Lax #-}

615encodeLatin1Lax :: (IsStream t, Monad m) => t m Char -> t m Word8

662encodeLatin1Lax :: (IsStream t, Monad m) => t m Char -> t m Word8

616−encodeLatin1Lax = S.map (fromIntegral . ord)

663+encodeLatin1Lax = encodeLatin1

617

664

618-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.

665-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.

619−-- The incoming stream is truncated if an invalid codepoint is encountered.

666+-- The function throws an error if an invalid codepoint is encountered.

620--

667--

621−-- /Since: 0.7.0/

668+-- @since 0.8.0

622−{-# INLINE decodeUtf8 #-}

669+{-# INLINE decodeUtf8' #-}

623−decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char

670+decodeUtf8' :: (Monad m, IsStream t) => t m Word8 -> t m Char

624−decodeUtf8 = D.fromStreamD . decodeUtf8D . D.toStreamD

671+decodeUtf8' = D.fromStreamD . decodeUtf8D' . D.toStreamD

625

672

626-- |

673-- |

627--

674--

628-- /Internal/

675-- /Internal/

629−{-# INLINE decodeUtf8Arrays #-}

676+{-# INLINE decodeUtf8Arrays' #-}

630−decodeUtf8Arrays :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char

677+decodeUtf8Arrays' :: (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char

631−decodeUtf8Arrays = D.fromStreamD . decodeUtf8ArraysD . D.toStreamD

678+decodeUtf8Arrays' = D.fromStreamD . decodeUtf8ArraysD' . D.toStreamD

632

679

633-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.

680-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.

634-- Any invalid codepoint encountered is replaced with the unicode replacement

681-- Any invalid codepoint encountered is replaced with the unicode replacement

635-- character.

682-- character.

636--

683--

637−-- /Since: 0.7.0/

684+-- /Since: 0.7.0 ("Streamly.Data.Unicode.Stream")/

685+--

686+-- /Since: 0.8.0 (Lenient Behaviour)/

687+{-# INLINE decodeUtf8 #-}

688+decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char

689+decodeUtf8 = D.fromStreamD . decodeUtf8D . D.toStreamD

690+

691+-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.

692+-- Any invalid codepoint encountered is dropped.

693+--

694+-- @since 0.8.0

695+{-# INLINE decodeUtf8_ #-}

696+decodeUtf8_ :: (Monad m, IsStream t) => t m Word8 -> t m Char

697+decodeUtf8_ = D.fromStreamD . decodeUtf8D_ . D.toStreamD

698+

699+-- | Same as 'decodeUtf8'

700+--

701+{-# DEPRECATED decodeUtf8Lax "Please use 'decodeUtf8' instead" #-}

638{-# INLINE decodeUtf8Lax #-}

702{-# INLINE decodeUtf8Lax #-}

639−decodeUtf8Lax :: (Monad m, IsStream t) => t m Word8 -> t m Char

703+decodeUtf8Lax :: (IsStream t, Monad m) => t m Word8 -> t m Char

640−decodeUtf8Lax = D.fromStreamD . decodeUtf8LenientD . D.toStreamD

704+decodeUtf8Lax = decodeUtf8

641

705

642-- |

706-- |

643--

707--

@@ -663,25 +727,36 @@ resumeDecodeUtf8Either st cp =

663-- |

727-- |

664--

728--

665-- /Internal/

729-- /Internal/

666−{-# INLINE decodeUtf8ArraysLenient #-}

730+{-# INLINE decodeUtf8Arrays #-}

667−decodeUtf8ArraysLenient ::

731+decodeUtf8Arrays ::

668 (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char

732 (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char

669−decodeUtf8ArraysLenient =

733+decodeUtf8Arrays =

670− D.fromStreamD . decodeUtf8ArraysLenientD . D.toStreamD

734+ D.fromStreamD . decodeUtf8ArraysD . D.toStreamD

671

735

672−-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.

736+-- |

673--

737--

674−-- /Since: 0.7.0/

738+-- /Internal/

739+{-# INLINE decodeUtf8Arrays_ #-}

740+decodeUtf8Arrays_ ::

741+ (MonadIO m, IsStream t) => t m (Array Word8) -> t m Char

742+decodeUtf8Arrays_ =

743+ D.fromStreamD . decodeUtf8ArraysD_ . D.toStreamD

744+

745+-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream. When

746+-- any invalid character (U+D800-U+D8FF) is encountered in the input stream the

747+-- function errors out.

748+--

749+-- @since 0.8.0

675−{-# INLINE encodeUtf8 #-}

750+{-# INLINE encodeUtf8' #-}

676−encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8

751+encodeUtf8' :: (Monad m, IsStream t) => t m Char -> t m Word8

677−encodeUtf8 = D.fromStreamD . encodeUtf8D . D.toStreamD

752+encodeUtf8' = D.fromStreamD . encodeUtf8D' . D.toStreamD

678

753

679-- | See section "3.9 Unicode Encoding Forms" in

754-- | See section "3.9 Unicode Encoding Forms" in

680-- https://www.unicode.org/versions/Unicode13.0.0/UnicodeStandard-13.0.pdf

755-- https://www.unicode.org/versions/Unicode13.0.0/UnicodeStandard-13.0.pdf

681--

756--

682−{-# INLINE_NORMAL encodeUtf8LaxD #-}

757+{-# INLINE_NORMAL encodeUtf8D #-}

683−encodeUtf8LaxD :: Monad m => Stream m Char -> Stream m Word8

758+encodeUtf8D :: Monad m => Stream m Char -> Stream m Word8

684−encodeUtf8LaxD (Stream step state) = Stream step' (EncodeState state WNil)

759+encodeUtf8D (Stream step state) = Stream step' (EncodeState state WNil)

685 where

760 where

686 {-# INLINE_LATE step' #-}

761 {-# INLINE_LATE step' #-}

687 step' gst (EncodeState st WNil) = do

762 step' gst (EncodeState st WNil) = do

@@ -710,10 +785,51 @@ encodeUtf8LaxD (Stream step state) = Stream step' (EncodeState state WNil)

710-- Invalid characters (U+D800-U+D8FF) in the input stream are replaced by the

785-- Invalid characters (U+D800-U+D8FF) in the input stream are replaced by the

711-- Unicode replacement character U+FFFD.

786-- Unicode replacement character U+FFFD.

712--

787--

713−-- /Since: 0.8.0/

788+-- /Since: 0.7.0 ("Streamly.Data.Unicode.Stream")/

789+--

790+-- /Since: 0.8.0 (Lenient Behaviour)/

791+{-# INLINE encodeUtf8 #-}

792+encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8

793+encodeUtf8 = D.fromStreamD . encodeUtf8D . D.toStreamD

794+

795+{-# INLINE_NORMAL encodeUtf8D_ #-}

796+encodeUtf8D_ :: Monad m => Stream m Char -> Stream m Word8

797+encodeUtf8D_ (Stream step state) = Stream step' (EncodeState state WNil)

798+ where

799+ {-# INLINE_LATE step' #-}

800+ step' gst (EncodeState st WNil) = do

801+ r <- step (adaptState gst) st

802+ return $

803+ case r of

804+ Yield c s ->

805+ case ord c of

806+ x | x <= 0x7F ->

807+ Yield (fromIntegral x) (EncodeState s WNil)

808+ | x <= 0x7FF -> Skip (EncodeState s (ord2 c))

809+ | x <= 0xFFFF ->

810+ if isSurrogate c

811+ then Skip $

812+ EncodeState s WNil

813+ else Skip (EncodeState s (ord3 c))

814+ | otherwise -> Skip (EncodeState s (ord4 c))

815+ Skip s -> Skip (EncodeState s WNil)

816+ Stop -> Stop

817+ step' _ (EncodeState s (WCons x xs)) = return $ Yield x (EncodeState s xs)

818+

819+-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream. Any

820+-- Invalid characters (U+D800-U+D8FF) in the input stream are dropped.

821+--

822+-- @since 0.8.0

823+{-# INLINE encodeUtf8_ #-}

824+encodeUtf8_ :: (Monad m, IsStream t) => t m Char -> t m Word8

825+encodeUtf8_ = D.fromStreamD . encodeUtf8D_ . D.toStreamD

826+

827+-- | Same as 'encodeUtf8'

828+--

829+{-# DEPRECATED encodeUtf8Lax "Please use 'encodeUtf8' instead" #-}

714{-# INLINE encodeUtf8Lax #-}

830{-# INLINE encodeUtf8Lax #-}

715−encodeUtf8Lax :: (Monad m, IsStream t) => t m Char -> t m Word8

831+encodeUtf8Lax :: (IsStream t, Monad m) => t m Char -> t m Word8

716−encodeUtf8Lax = D.fromStreamD . encodeUtf8LaxD . D.toStreamD

832+encodeUtf8Lax = encodeUtf8

717

833

718{-

834{-

719-------------------------------------------------------------------------------

835-------------------------------------------------------------------------------

src/Streamly/Unicode/Stream.hs+4−4

@@ -33,7 +33,7 @@

33-- stream as @Array Char@ using the array 'Streamly.Data.Array.Storable.Foreign.write' fold.

34-- The 'Array' type provides a more compact representation and pinned memory

35-- reducing GC overhead. If space efficiency is a concern you can use

36−-- 'encodeUtf8' on the 'Char' stream before writing it to an 'Array' providing

36+-- 'encodeUtf8'' on the 'Char' stream before writing it to an 'Array' providing

37-- an even more compact representation.

38--

39-- = String Literals

@@ -69,13 +69,13 @@ module Streamly.Unicode.Stream

69 -- * Construction (Decoding)

70 decodeLatin1

71 , decodeUtf8

72− , decodeUtf8Lax

72+ , decodeUtf8'

74 -- * Elimination (Encoding)

75 , encodeLatin1

76− , encodeLatin1Lax

76+ , encodeLatin1'

77 , encodeUtf8

78− , encodeUtf8Lax

78+ , encodeUtf8'

79 {-

80 -- * Operations on character strings

81 , strip -- (dropAround isSpace)

test/Streamly/Test/FileSystem/Event.hs+1−1

@@ -36,7 +36,7 @@ import qualified Streamly.Internal.Data.Array.Storable.Foreign as Array

36-------------------------------------------------------------------------------

38toUtf8 :: MonadIO m => String -> m (Array Word8)

39−toUtf8 = Array.fromStream . Unicode.encodeUtf8 . Stream.fromList

39+toUtf8 = Array.fromStream . Unicode.encodeUtf8' . Stream.fromList

41-------------------------------------------------------------------------------

42-- Main

test/String.hs+12−12

@@ -33,29 +33,29 @@ genUnicode = listOf arbitraryUnicodeChar

33genWord8 :: Gen [Word8]

34genWord8 = listOf arbitrary

36−propDecodeEncodeId :: Property

36+propDecodeEncodeId' :: Property

37−propDecodeEncodeId =

37+propDecodeEncodeId' =

38 forAll genUnicode $ \list ->

39 monadicIO $ do

40− let wrds = SS.encodeUtf8 $ S.fromList list

40+ let wrds = SS.encodeUtf8' $ S.fromList list

41− chrs <- S.toList $ SS.decodeUtf8 wrds

41+ chrs <- S.toList $ SS.decodeUtf8' wrds

42 assert (chrs == list)

44-- XXX need to use invalid characters

45−propDecodeEncodeIdLenient :: Property

45+propDecodeEncodeId :: Property

46−propDecodeEncodeIdLenient =

46+propDecodeEncodeId =

47 forAll genUnicode $ \list ->

48 monadicIO $ do

49 let wrds = SS.encodeUtf8 $ S.fromList list

50− chrs <- S.toList $ SS.decodeUtf8Lax wrds

50+ chrs <- S.toList $ SS.decodeUtf8 wrds

51 assert (chrs == list)

53propDecodeEncodeIdArrays :: Property

54propDecodeEncodeIdArrays =

55 forAll genUnicode $ \list ->

56 monadicIO $ do

57− let wrds = SS.encodeUtf8 $ S.fromList list

57+ let wrds = SS.encodeUtf8' $ S.fromList list

58− chrs <- S.toList $ IUS.decodeUtf8ArraysLenient

58+ chrs <- S.toList $ IUS.decodeUtf8Arrays

59 (S.fold A.write wrds)

60 assert (chrs == list)

@@ -117,9 +117,9 @@ main = hspec

117 $ modifyMaxSuccess (const 1000)

118 $ do

119 describe "UTF8 - Encoding / Decoding" $ do

120− prop "decodeUtf8 . encodeUtf8 == id" $ propDecodeEncodeId

120+ prop "decodeUtf8' . encodeUtf8' == id" $ propDecodeEncodeId'

121− prop "decodeUtf8Lax . encodeUtf8 == id" $ propDecodeEncodeIdLenient

121+ prop "decodeUtf8 . encodeUtf8' == id" $ propDecodeEncodeId

122− prop "decodeUtf8ArraysLenient . encodeUtf8 == id"

122+ prop "decodeUtf8Arrays . encodeUtf8' == id"

123 $ propDecodeEncodeIdArrays

124 prop "Streamly.Data.String.lines == Prelude.lines" $ testLines

125 prop "Arrays Streamly.Data.String.lines == Prelude.lines" $ testLinesArray