From 369274045d30b148af4372747e3641784c72ae69 Mon Sep 17 00:00:00 2001
From: Adithya Kumar <adithya@composewell.com>
Date: Tue, 25 Jul 2023 01:38:07 +0530
Subject: [PATCH] Expose incremental UTF8 decoding APIs

---
 core/src/Streamly/Internal/Unicode/Stream.hs | 32 +++++++++++++++++---
 core/src/Streamly/Unicode/Stream.hs          |  7 +++++
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/core/src/Streamly/Internal/Unicode/Stream.hs b/core/src/Streamly/Internal/Unicode/Stream.hs
index f6847bf7d..0cbfd73b6 100644
--- a/core/src/Streamly/Internal/Unicode/Stream.hs
+++ b/core/src/Streamly/Internal/Unicode/Stream.hs
@@ -190,8 +190,23 @@ encodeLatin1Lax = encodeLatin1
 -- UTF-8 decoding
 -------------------------------------------------------------------------------
 
--- Int helps in cheaper conversion from Int to Char
+-- | CodePoint represents a specific character in the Unicode standard.
+--
+-- It is meant to be used with the resumable decoding APIs such as
+-- 'resumeDecodeUtf8Either'.
+--
+-- On decoding failure we return the current 'CodePoint' and the 'DecodeState'
+-- in 'DecodeError'.
 type CodePoint = Int
+
+-- | DecodeState refers to the number of bytes remaining to complete the current
+-- UTF-8 character decoding.
+--
+-- It is meant to be used with the resumable decoding APIs such as
+-- 'resumeDecodeUtf8Either'.
+--
+-- On decoding failure we return the current 'CodePoint' and the 'DecodeState'
+-- in 'DecodeError'.
 type DecodeState = Word8
 
 -- We can divide the errors in three general categories:
@@ -410,17 +425,24 @@ decodeUtf8EitherD :: Monad m
     => D.Stream m Word8 -> D.Stream m (Either DecodeError Char)
 decodeUtf8EitherD = resumeDecodeUtf8EitherD 0 0
 
--- |
+-- | Decode a bytestream as UTF-8 encoded characters, returning an 'Either'
+-- stream.
+--
+-- This function is similar to 'decodeUtf8', but instead of replacing the
+-- invalid codepoint encountered, it returns a 'Left' 'DecodeError'.
+--
+-- When decoding is successful and a valid character is encountered, the
+-- function returns 'Right Char'.
 --
--- /Pre-release/
 {-# INLINE decodeUtf8Either #-}
 decodeUtf8Either :: Monad m
     => Stream m Word8 -> Stream m (Either DecodeError Char)
 decodeUtf8Either = decodeUtf8EitherD
 
--- |
+-- | Resuming the decoding of a bytestream given a 'DecodeState' and a
+-- 'CodePoint'.
 --
--- /Pre-release/
+-- >>> decodeUtf8Either = resumeDecodeUtf8Either 0 0
 {-# INLINE resumeDecodeUtf8Either #-}
 resumeDecodeUtf8Either
     :: Monad m
diff --git a/core/src/Streamly/Unicode/Stream.hs b/core/src/Streamly/Unicode/Stream.hs
index c76682120..8cedb9350 100644
--- a/core/src/Streamly/Unicode/Stream.hs
+++ b/core/src/Streamly/Unicode/Stream.hs
@@ -81,6 +81,13 @@ module Streamly.Unicode.Stream
     , decodeUtf8'
     , decodeUtf8Chunks
 
+    -- ** Resumable UTF-8 Decoding
+    , DecodeError(..)
+    , DecodeState
+    , CodePoint
+    , decodeUtf8Either
+    , resumeDecodeUtf8Either
+
     -- * Elimination (Encoding)
     , encodeLatin1
     , encodeLatin1'