bcf0ab1...master · streamly

git.iko.soy/streamly

188 files changed+9907−5786

since splitsnapshots

unifiedsplit

.circleci/config.yml+73−277

@@ -1,7 +1,7 @@

11version: 2.1

33#-----------------------------------------------------------------------------

4−# packcheck-0.4.2

4+# packcheck-0.7.0

55# Packcheck global environment variables

66#-----------------------------------------------------------------------------

@@ -11,6 +11,8 @@ env: &env

1111 # Common options

1212 # ------------------------------------------------------------------------

1313 # GHC_OPTIONS: "-Werror"

14+ # For updating see: https://downloads.haskell.org/~ghcup/

15+ GHCUP_VERSION: 0.1.20.0

1416 CABAL_REINIT_CONFIG: "y"

1517 LC_ALL: "C.UTF-8"

1618

@@ -22,7 +24,6 @@ env: &env

2224 # DISABLE_DOCS: "y"

2325 # DISABLE_SDIST_BUILD: "y"

2426 # DISABLE_DIST_CHECKS: "y"

25− ENABLE_INSTALL: "y"

2627

2728 # ------------------------------------------------------------------------

2829 # stack options

@@ -38,15 +39,12 @@ env: &env

3839 # cabal options

3940 # ------------------------------------------------------------------------

4041 CABAL_CHECK_RELAX: "y"

41− CABAL_NO_SANDBOX: "y"

42− CABAL_HACKAGE_MIRROR: "hackage.haskell.org:http://hackage.fpcomplete.com"

4342 CABAL_BUILD_OPTIONS: "-j1 --flag limit-build-mem"

4443

4544 # ------------------------------------------------------------------------

4645 # Where to find the required tools

4746 # ------------------------------------------------------------------------

48− PATH: /opt/ghc/bin:/opt/ghcjs/bin:/sbin:/usr/sbin:/bin:/usr/bin

49− TOOLS_DIR: /opt

47+ PATH: /sbin:/usr/sbin:/bin:/usr/bin

5048

5149 # ------------------------------------------------------------------------

5250 # Location of packcheck.sh (the shell script invoked to perform CI tests ).

@@ -59,15 +57,15 @@ env: &env

5957 # If you have not committed packcheck.sh in your repo at PACKCHECK

6058 # then it is automatically pulled from this URL.

6159 PACKCHECK_GITHUB_URL: "https://raw.githubusercontent.com/composewell/packcheck"

62− PACKCHECK_GITHUB_COMMIT: "a302772c8f3791c477edd1764b82f8d1f5b22fdf"

60+ PACKCHECK_GITHUB_COMMIT: "f6cfd7b7a33b4bf6fd63cce6a3fb3d26f5031403"

6361

6462executors:

6563 amd64-executor:

6664 docker:

67− - image: ubuntu:xenial

65+ - image: ubuntu:latest

6866 x86-executor:

6967 docker:

70− - image: i386/ubuntu:eoan

68+ - image: i386/ubuntu:focal

7169

7270#-----------------------------------------------------------------------------

7371# Common utility stuff, not to be modified usually

@@ -79,38 +77,50 @@ preinstall: &preinstall

7977 # required for https/cache save and restore

8078 apt-get install -y ca-certificates

8179

82− # required to (re)generate the configure script

83− apt-get install -y autoconf

84−

85− # For ghc and cabal-install packages from hvr's ppa

80+ # Alternative way of installing ghc and cabal, directly from

81+ # haskell.org instead of using ghcup. NOTE: this is for Debian

82+ # only and is debian release specific.

8683 # gnupg is required for apt-key to work

87− apt-get install -y gnupg

88− apt-get install -y apt-transport-https

89− apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA3CBA3FFE22B574

90− apt-get install -y software-properties-common

91− add-apt-repository -y ppa:hvr/ghc

92− # echo "deb http://downloads.haskell.org/debian stretch main" >> /etc/apt/sources.list

93− echo "deb-src http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main" >> /etc/apt/sources.list

94− apt-get update

84+ #apt-get install -y gnupg

85+ #apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA3CBA3FFE22B574

86+ #echo "deb http://downloads.haskell.org/debian buster main" >> /etc/apt/sources.list

87+ #apt-get update

9588

96− # required for packcheck

97− apt-get install -y curl

89+ # required for outbound https for stack and for stack setup

90+ apt-get install -y netbase xz-utils make

91+ apt-get install -y zlib1g-dev

9892

99− # required for circleci.

100− apt-get install -y libcurl4-gnutls-dev

101− apt-get install -y git

93+ # For ghcup to install ghc

94+ if test -n "$GHCUP_VERSION"

95+ then

96+ apt-get install -y gcc

97+ apt-get install -y g++

98+ fi

99+

100+ # libgmp required by ghc for linking

101+ apt-get install -y libgmp-dev

102102 apt-get install -y libtinfo-dev

103103

104− # required for outbound https for stack and for stack setup

105− apt-get install -y netbase xz-utils make

104+ # Required by cabal when git URL is specified in project file

105+ apt-get install -y git

106+

107+ # Required for and by packcheck

108+ apt-get install -y curl

106109

107−project-preinstall: &project-preinstall

108− run: |

109110 # Get packcheck if needed

110− CURL=$(which curl)

111− PACKCHECK_URL=${PACKCHECK_GITHUB_URL}/${PACKCHECK_GITHUB_COMMIT}/packcheck.sh

112− if test ! -e "$PACKCHECK"; then $CURL -sL -o "$PACKCHECK" $PACKCHECK_URL; fi;

113− chmod +x $PACKCHECK

111+ if test ! -e "$PACKCHECK"

112+ then

113+ if test -z "$PACKCHECK_GITHUB_COMMIT"

114+ then

115+ die "PACKCHECK_GITHUB_COMMIT is not specified."

116+ fi

117+ PACKCHECK_URL=${PACKCHECK_GITHUB_URL}/${PACKCHECK_GITHUB_COMMIT}/packcheck.sh

118+ curl --fail -sL -o "$PACKCHECK" $PACKCHECK_URL || exit 1

119+ chmod +x $PACKCHECK

120+ elif test ! -x "$PACKCHECK"

121+ then

122+ chmod +x $PACKCHECK

123+ fi

114124

115125restore: &restore

116126 # Needs to happen after installing ca-certificates

@@ -126,271 +136,71 @@ save: &save

126136 save_cache:

127137 key: v1-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}

128138 paths:

129− - ~/.cabal

130− - ~/.ghc

131139 - ~/.local

140+ - ~/.cabal

132141 - ~/.stack

142+ - ~/.ghcup

133143

134144#-----------------------------------------------------------------------------

135145# Build matrix

136146#-----------------------------------------------------------------------------

137147

138148jobs:

139− cabal-ghc-8_10_4:

149+ cabal-ghc-8_6_5:

140150 <<: *env

141151 executor: amd64-executor

142152 steps:

143− - *preinstall

144153 - checkout

145− - *project-preinstall

146154 - *restore

155+ - *preinstall

147156 - run:

148157 environment:

158+ GHCVER: "8.6.5"

159+ CABAL_VERSION: "3.6.2.0"

149160 CABAL_PROJECT: "cabal.project"

150− DISABLE_TEST: "yes"

151− DISABLE_BENCH: "yes"

152161 DISABLE_SDIST_BUILD: "yes"

162+ CABAL_BUILD_OPTIONS: "--flag debug --flag -opt"

153163 command: |

154− apt-get install -y ghc-8.10.4

155− apt-get install -y cabal-install-3.4

156164 bash -c "$PACKCHECK cabal"

157165 - *save

158− stack-ghc-8_8:

166+ cabal-ghc-9_8_1-docspec:

159167 <<: *env

160168 executor: amd64-executor

161169 steps:

162− - *preinstall

163170 - checkout

164− - *project-preinstall

165171 - *restore

166− - run:

167− name: GHC 8.8 + stack lts-16.12

168− environment:

169− BUILD: "stack"

170− RESOLVER: "lts-16.12"

171− SDIST_OPTIONS: "--ignore-check"

172− STACK_BUILD_OPTIONS: "--flag streamly-benchmarks:-opt"

173− command: |

174− apt-get build-dep -y ghc-8.8.4

175− apt-get install -y cabal-install-3.2

176− bash -c "$PACKCHECK $BUILD"

177− - *save

178− cabal-ghc-8_6_5:

179− <<: *env

180− executor: amd64-executor

181− steps:

182172 - *preinstall

183− - checkout

184− - *project-preinstall

185− - *restore

186− - run: |

187− apt-get install -y ghc-8.6.3

188− apt-get install -y cabal-install-2.4

189− bash -c "$PACKCHECK cabal-v2"

190− - *save

191− cabal-ghc-8_4_4:

192− <<: *env

193− executor: x86-executor

194− steps:

195− - *preinstall

196− - checkout

197− - *project-preinstall

198− - *restore

199− - run:

200− name: GHC 8.4.4 + x86 + debug

201− environment:

202− BUILD: "cabal-v2"

203− GHCVER: "8.4.4"

204− CABAL_BUILD_OPTIONS: "--flag debug --flag limit-build-mem"

205− command: |

206− apt-get install -y ghc-8.4.4

207− apt-get install -y cabal-install-3.2

208− bash -c "$PACKCHECK $BUILD"

209− - *save

210− cabal-ghc-8_2_2:

211− <<: *env

212− executor: amd64-executor

213− steps:

214− - *preinstall

215− - checkout

216− - *project-preinstall

217− - *restore

218173 - run:

219174 environment:

220− BUILD: "cabal-v2"

221− DISABLE_TEST: "yes"

222− DISABLE_BENCH: "yes"

223− DISABLE_DOCS: "yes"

224− DISABLE_SDIST_BUILD: "yes"

225− DISABLE_DIST_CHECKS: "yes"

175+ GHCVER: "9.8.1"

176+ CABAL_VERSION: "3.10.1.0"

177+ CABAL_PROJECT: "cabal.project.doctest"

178+ DISABLE_SDIST_BUILD: "y"

179+ DISABLE_TEST: "y"

180+ DISABLE_BENCH: "y"

181+ DISABLE_DOCS: "y"

182+ ENABLE_DOCSPEC: "y"

183+ DOCSPEC_URL: https://github.com/phadej/cabal-extras/releases/download/cabal-docspec-0.0.0.20210111/cabal-docspec-0.0.0.20210111.xz

184+ DOCSPEC_OPTIONS: "--timeout 60 --check-properties --property-variables xs"

226185 command: |

227− apt-get install -y ghc-8.2.2

228− apt-get install -y cabal-install-3.2

229− bash -c "$PACKCHECK $BUILD"

230− - *save

231− cabal-ghc-8_0_2:

232− <<: *env

233− executor: amd64-executor

234− steps:

235− - *preinstall

236− - checkout

237− - *project-preinstall

238− - *restore

239− - run:

240− environment:

241− BUILD: "cabal-v2"

242− DISABLE_TEST: "yes"

243− DISABLE_BENCH: "yes"

244− DISABLE_DOCS: "yes"

245− DISABLE_SDIST_BUILD: "yes"

246− DISABLE_DIST_CHECKS: "yes"

247− command: |

248− apt-get install -y ghc-8.0.2

249− apt-get install -y cabal-install-3.2

250− bash -c "$PACKCHECK $BUILD"

251− - *save

252− cabal-ghc-7_10_3:

253− <<: *env

254− executor: amd64-executor

255− steps:

256− - *preinstall

257− - checkout

258− - *project-preinstall

259− - *restore

260− - run: |

261− apt-get install -y ghc-7.10.3

262− apt-get install -y cabal-install-2.4

263− bash -c "$PACKCHECK cabal-v2"

186+ bash -c "$PACKCHECK cabal"

264187 - *save

265− stack-ghc-8_4:

188+ hlint-trailing-spaces:

266189 <<: *env

267190 executor: amd64-executor

268191 steps:

269− - *preinstall

270192 - checkout

271− - *project-preinstall

272193 - *restore

273− - run: |

274− bash -c "$PACKCHECK stack RESOLVER=lts-12"

275− - *save

276− stack-ghc-8_2:

277− <<: *env

278− executor: amd64-executor

279− steps:

280194 - *preinstall

281− - checkout

282− - *project-preinstall

283− - *restore

284− - run: |

285− bash -c "$PACKCHECK stack RESOLVER=lts-11"

286− - *save

287− cabal-ghcjs-8_4:

288− <<: *env

289− executor: amd64-executor

290− steps:

291− - *preinstall

292− - checkout

293− - *project-preinstall

294− - *restore

295195 - run:

296− name: GHCJS 8.4+no-test+no-docs

196+ name: hlint and trailing spaces

297197 environment:

298− BUILD: cabal-v2

299− ENABLE_GHCJS: "yes"

300− DISABLE_TEST: "yes"

301− ENABLE_INSTALL: ""

302− DISABLE_DOCS: "yes"

303198 DISABLE_SDIST_BUILD: "yes"

304− command: |

305− add-apt-repository -y ppa:hvr/ghcjs

306− add-apt-repository -y 'deb https://deb.nodesource.com/node_11.x xenial main'

307− curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -

308− apt-get update

309− apt-get install -y cabal-install-3.2

310− apt-get install -y nodejs

311− apt-get install -y ghcjs-8.4

312− cabal user-config update -a "jobs: 1"

313− bash -c "$PACKCHECK $BUILD"

314− - *save

315− coveralls-ghc-8_2_2:

316− <<: *env

317− executor: amd64-executor

318− steps:

319− - *preinstall

320− - checkout

321− - *project-preinstall

322− - *restore

323− - run:

324− environment:

325− BUILD: cabal

326− COVERALLS_OPTIONS: "--repo-token=\"$REPO_TOKEN\" --coverage-mode=StrictlyFullLines --exclude-dir=test test"

327− GHC_OPTIONS: "-DCOVERAGE_BUILD"

328− command: |

329− apt-get install -y ghc-8.2.2

330− apt-get install -y cabal-install-2.4

331− bash -c "$PACKCHECK cabal-v1"

332− - *save

333− coveralls-ghc-8_8_3:

334− <<: *env

335− executor: amd64-executor

336− steps:

337− - *preinstall

338− - checkout

339− - *project-preinstall

340− - *restore

341− - run:

342− environment:

343− BUILD: cabal-v2

344− COVERALLS_OPTIONS: "--repo-token=\"$REPO_TOKEN\" --coverage-mode=StrictlyFullLines --exclude-dir=test"

345− CABAL_PROJECT: "cabal.project.coverage"

346− CABAL_BUILD_TARGETS: ""

347− command: |

348− apt-get install -y ghc-8.8.3

349− apt-get install -y cabal-install-3.0

350− bash -c "$PACKCHECK cabal-v2"

351− no_output_timeout: 25m

352− - *save

353− coveralls-ghc-8_10_2:

354− <<: *env

355− executor: amd64-executor

356− steps:

357− - *preinstall

358− - checkout

359− - *project-preinstall

360− - *restore

361− - run:

362− name: Build and test with inspection, coverage, Werror

363− command: |

364− apt-get install -y ghc-8.10.2

365− apt-get install -y cabal-install-3.2

366− # Run tests with coverage

367− cabal update

368− cabal user-config update -a "jobs: 1"

369− bin/test.sh --coverage --raw

370− # Upload results to coveralls.io

371− PATH=$HOME/.cabal/bin:$PATH

372− export PATH

373− which hpc-coveralls 2>/dev/null || cabal install --project-file cabal.project.hpc-coveralls hpc-coveralls

374− hpc-coveralls --repo-token="$REPO_TOKEN" --coverage-mode=StrictlyFullLines

375− no_output_timeout: 25m

376− - *save

377− hlint-src:

378− <<: *env

379− executor: amd64-executor

380− steps:

381− - *preinstall

382− - checkout

383− - *project-preinstall

384− - *restore

385− - run:

386− name: Hlint src

387− environment:

388− BUILD: hlint

199+ HLINT_VERSION: 3.6.1

389200 HLINT_OPTIONS: lint

390− DISABLE_SDIST_BUILD: "yes"

391201 HLINT_TARGETS: core/src src test benchmark

392202 command: |

393− bash -c "$PACKCHECK $BUILD" || exit 1

203+ bash -c "$PACKCHECK hlint" || exit 1

394204 echo "Checking trailing spaces..."

395205 count=$(find . -name "*.hs" -exec grep -H '\ $' {} \; | tee /dev/tty | wc -l)

396206 exit $count

@@ -400,23 +210,9 @@ workflows:

400210 version: 2

401211 build:

402212 jobs:

403− - cabal-ghc-8_10_4:

404− name: GHC 8.10.4 + sdist + no-test + no-bench

405− #- cabal-ghc-8.6.5

406− #- cabal-ghc-8_4_4:

407− # name: GHC 8.4.4 + x86 + debug

408− #- cabal-ghc-8_2_2:

409− # name: GHC 8.2.2 + no-test + no-bench + no-docs

410− #- cabal-ghc-8_0_2:

411− # name: GHC 8.0.2 + no-test + no-bench + no-docs

412− #- cabal-ghc-7.10.3

413− # - cabal-ghcjs-8_4:

414− # name: GHCJS 8.4 + no-test + no-docs

415− #- stack-ghc-8.4

416− #- stack-ghc-8.2

417− #- coveralls-ghc-8.2.2

418− #- coveralls-ghc-8.8.3

419− #- coveralls-ghc-8_10_2:

420− # name: GHC 8.10.2 + inspection + coverage + Werror

421− - hlint-src:

422− name: Hlint src + Trailing Spaces

213+ #- cabal-ghc-8_6_5:

214+ # name: 8.6.5-debug-unoptimized

215+ - cabal-ghc-9_8_1-docspec:

216+ name: ghc-9.8.1-docspec

217+ - hlint-trailing-spaces:

218+ name: hlint and trailing spaces

.github/workflows/haskell.yml+61−60

@@ -13,9 +13,15 @@ jobs:

1313 # packcheck environment variables

1414 LC_ALL: C.UTF-8

1515 BUILD: ${{ matrix.build }}

16+ # For updating see: https://downloads.haskell.org/~ghcup/

17+ GHCUP_VERSION: 0.1.20.0

18+ DOCSPEC_URL: https://github.com/phadej/cabal-extras/releases/download/cabal-docspec-0.0.0.20210111/cabal-docspec-0.0.0.20210111.xz

19+ DOCSPEC_OPTIONS: "--timeout 60 --check-properties --property-variables xs"

20+ GHCUP_GHC_OPTIONS: ${{ matrix.ghcup_ghc_options }}

1621 GHCVER: ${{ matrix.ghc_version }}

1722 CABALVER: ${{ matrix.cabal_version }}

1823 DISABLE_DOCS: ${{ matrix.disable_docs }}

24+ ENABLE_DOCSPEC: ${{ matrix.enable_docspec }}

1925 DISABLE_TEST: ${{ matrix.disable_test }}

2026 DISABLE_BENCH: ${{ matrix.disable_bench }}

2127 DISABLE_DIST_CHECKS: ${{ matrix.disable_dist_checks }}

@@ -39,14 +45,18 @@ jobs:

3945 STACK_BUILD_OPTIONS: ${{ matrix.stack_build_options }}

4046

4147 # packcheck location and revision

42− PACKCHECK_LOCAL_PATH: "./packcheck.sh"

48+ PACKCHECK: "./packcheck.sh"

4349 PACKCHECK_GITHUB_URL: "https://raw.githubusercontent.com/composewell/packcheck"

44− PACKCHECK_GITHUB_COMMIT: "a302772c8f3791c477edd1764b82f8d1f5b22fdf"

50+ PACKCHECK_GITHUB_COMMIT: "v0.7.0"

4551

4652 # Pull token from "secrets" setting of the github repo

4753 COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }}

4854 COVERAGE: ${{ matrix.coverage }}

49− DOCTEST: ${{ matrix.doctest }}

55+

56+ # hlint

57+ HLINT_VERSION: 3.6.1

58+ HLINT_OPTIONS: "lint"

59+ HLINT_TARGETS: "core/src src test benchmark"

5060

5161 # Subdir

5262 SUBDIR: ${{ matrix.subdir }}

@@ -59,7 +69,7 @@ jobs:

5969 # The order is important to optimize fail-fast.

6070 name:

6171 - 9.8.1-Werror

62− - 9.2.8-docspec

72+ # - 9.8.1-docspec

6373 # - 8.10.7-coverage

6474

6575 # Note: if cabal.project is not specified benchmarks and tests won't

@@ -68,9 +78,19 @@ jobs:

6878 include:

6979 - name: head

7080 ghc_version: head

81+ # The URL may change, to find a working URL go to https://gitlab.haskell.org/ghc/ghc/-/jobs/

82+ # Find a debian10/11/12 job, click on a passed/failed status, at the

83+ # end of the output you will find the tar.xz name, put that tar

84+ # name after "raw/", and put the job name after "job=".

85+ # Also see https://github.com/mpickering/ghc-artefact-nix/blob/master/gitlab-artifact.nix

86+ #

87+ # May also use ghcup for installing ghc head version, use the

88+ # version "LatestNightly", and the following config:

89+ # ghcup config add-release-channel https://ghc.gitlab.haskell.org/ghcup-metadata/ghcup-nightlies-0.0.7.yaml

90+ ghcup_ghc_options: "-u https://gitlab.haskell.org/ghc/ghc/-/jobs/artifacts/master/raw/ghc-x86_64-linux-deb10-int_native-validate.tar.xz?job=x86_64-linux-deb10-int_native-validate"

7191 runner: ubuntu-latest

7292 build: cabal

73− cabal_version: 3.10.1.0

93+ cabal_version: 3.10.2.0

7494 cabal_project: cabal.project.ghc-head

7595 disable_sdist_build: "y"

7696 ignore_error: true

@@ -83,6 +103,18 @@ jobs:

83103 disable_sdist_build: "y"

84104 cabal_project: cabal.project.Werror

85105 ignore_error: false

106+ #- name: 9.8.1-docspec

107+ # ghc_version: 9.8.1

108+ # runner: ubuntu-latest

109+ # build: cabal

110+ # cabal_version: 3.10.1.0

111+ # cabal_project: cabal.project.doctest

112+ # disable_test: "y"

113+ # disable_bench: "y"

114+ # disable_docs: "y"

115+ # enable_docspec: "y"

116+ # disable_sdist_build: "y"

117+ # ignore_error: false

86118 - name: 9.8.1-fusion-inspection

87119 ghc_version: 9.8.1

88120 runner: ubuntu-latest

@@ -116,14 +148,6 @@ jobs:

116148 cabal_version: 3.6.2.0

117149 disable_sdist_build: "y"

118150 ignore_error: false

119− - name: 9.2.8-docspec

120− ghc_version: 9.2.8

121− runner: ubuntu-latest

122− doctest: "y"

123− cabal_version: 3.6.2.0

124− cabal_project: cabal.project

125− disable_sdist_build: "y"

126− ignore_error: false

127151 - name: 9.0.2-streamly-sdist

128152 ghc_version: 9.0.2

129153 runner: ubuntu-latest

@@ -170,66 +194,44 @@ jobs:

170194 cabal_build_options: "--flag debug --flag -opt"

171195 disable_sdist_build: "y"

172196 ignore_error: false

197+ - name: hlint

198+ build: hlint

199+ runner: ubuntu-latest

200+ ignore_error: true

173201

174202 steps:

175203 - uses: actions/checkout@v2

176204

177205 - uses: actions/cache@v1

178− name: Cache ~/.cabal

206+ name: Cache common directories

179207 with:

180− path: ~/.cabal

208+ path: |

209+ ~/.local

210+ ~/.cabal

211+ ~/.stack

212+ ~/.ghcup

181213 # Bump the key version to clear the cache

182214 key: ${{ runner.os }}-${{ matrix.ghc_version }}-cabal-v2

183215

184216 - name: Download packcheck

185217 run: |

186− # Get packcheck if needed

187− CURL=$(which curl)

188− PACKCHECK_URL=${PACKCHECK_GITHUB_URL}/${PACKCHECK_GITHUB_COMMIT}/packcheck.sh

189− if test ! -e "$PACKCHECK_LOCAL_PATH"; then $CURL -sL -o "$PACKCHECK_LOCAL_PATH" $PACKCHECK_URL; fi;

190− chmod +x $PACKCHECK_LOCAL_PATH

191−

192− os=$(uname -s -m)

193− case "$os" in

194− "Linux x86_64") GHCUP_ARCH="x86_64-linux" ;;

195− "Darwin x86_64") GHCUP_ARCH="x86_64-apple-darwin" ;;

196− *) echo "Unknown OS/Arch: $os"; exit 1;;

197− esac

198−

199− # Check available versions here: https://downloads.haskell.org/~ghcup/

200− GHCUP_VER=0.1.18.0

201− $CURL -sL -o ./ghcup https://downloads.haskell.org/~ghcup/$GHCUP_VER/${GHCUP_ARCH}-ghcup-$GHCUP_VER

202− chmod +x ./ghcup

203− if test "$GHCVER" = "head"

218+ if test ! -e "$PACKCHECK"

204219 then

205− # The URL may change, to find a working URL go to https://gitlab.haskell.org/ghc/ghc/-/jobs/

206− # Find a debian10 job, click on a passed/failed job, at the

207− # end of the output you will find the tar.xz name, put that after

208− # "raw/", and put the job name after "job=".

209− # Also see https://github.com/mpickering/ghc-artefact-nix/blob/master/gitlab-artifact.nix

210− ./ghcup install ghc -u https://gitlab.haskell.org/ghc/ghc/-/jobs/artifacts/master/raw/ghc-x86_64-linux-deb10-int_native-validate.tar.xz?job=x86_64-linux-deb10-int_native-validate head

211− else

212− ./ghcup install ghc $GHCVER

220+ if test -z "$PACKCHECK_GITHUB_COMMIT"

221+ then

222+ die "PACKCHECK_GITHUB_COMMIT is not specified."

223+ fi

224+ PACKCHECK_URL=${PACKCHECK_GITHUB_URL}/${PACKCHECK_GITHUB_COMMIT}/packcheck.sh

225+ curl --fail -sL -o "$PACKCHECK" $PACKCHECK_URL || exit 1

226+ chmod +x $PACKCHECK

227+ elif test ! -x "$PACKCHECK"

228+ then

229+ chmod +x $PACKCHECK

213230 fi

214− ./ghcup install cabal $CABALVER

215− ./ghcup set ghc $GHCVER

216− ./ghcup set cabal $CABALVER

217231

218232 - name: Run tests

219233 run: |

220− if test -n "$DOCTEST"

221− then

222− mkdir -p $HOME/.cabal/bin

223− curl -sL https://github.com/phadej/cabal-extras/releases/download/cabal-docspec-0.0.0.20210111/cabal-docspec-0.0.0.20210111.xz > cabal-docspec.xz

224− echo '0829bd034fba901cbcfe491d98ed8b28fd54f9cb5c91fa8e1ac62dc4413c9562 cabal-docspec.xz' | sha256sum -c -

225− xz -d < cabal-docspec.xz > $HOME/.cabal/bin/cabal-docspec

226− rm -f cabal-docspec.xz

227− chmod a+x $HOME/.cabal/bin/cabal-docspec

228− cabal update

229− cabal build --project-file cabal.project.doctest all

230− $HOME/.cabal/bin/cabal-docspec --version

231− $HOME/.cabal/bin/cabal-docspec --timeout 60 --check-properties --property-variables xs

232− elif test -n "$COVERAGE"

234+ if test -n "$COVERAGE"

233235 then

234236 # Run tests with coverage

235237 cabal update

@@ -242,16 +244,15 @@ jobs:

242244 # XXX Uncomment this and fix it properly later

243245 # hpc-coveralls --repo-token="$COVERALLS_TOKEN" --coverage-mode=StrictlyFullLines

244246 else

245− export TOOLS_DIR=/opt

246247 # /usr/local/opt/curl/bin for macOS

247248 export PATH=$HOME/.local/bin:$HOME/.ghcup/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/opt/curl/bin

248249 if test -n "$SUBDIR"

249250 then

250− mv "$PACKCHECK_LOCAL_PATH" "$SUBDIR/packcheck.sh"

251+ mv "$PACKCHECK" "$SUBDIR/packcheck.sh"

251252 # This is required as dist-newstyle will be created on the top level

252253 # directory as it is considered the working dir.

253254 rm cabal.project

254255 cd "$SUBDIR"

255256 fi

256− bash -c "$PACKCHECK_LOCAL_PATH $BUILD"

257+ bash -c "$PACKCHECK $BUILD"

257258 fi

.hlint.ignore+5

@@ -2,6 +2,9 @@ core/src/Streamly/Internal/Data/Stream/StreamK/Type.hs

22core/src/Streamly/Internal/Data/Pipe/Type.hs

33core/src/Streamly/Internal/Unicode/Stream.hs

44src/Streamly/Internal/Data/SmallArray/Type.hs

5+src/Streamly/Internal/Data/Stream/IsStream/Eliminate.hs

6+src/Streamly/Internal/Data/Stream/Serial.hs

7+src/Streamly/Internal/Data/Stream/Zip.hs

58test/Streamly/Test/Data/Array.hs

69test/Streamly/Test/Data/Parser.hs

710test/Streamly/Test/Data/ParserK.hs

@@ -10,6 +13,7 @@ test/Streamly/Test/Data/Unfold.hs

1013test/Streamly/Test/FileSystem/Event.hs

1114test/Streamly/Test/Prelude/Concurrent.hs

1215test/Streamly/Test/Data/Stream/Concurrent.hs

16+test/lib/Streamly/Test/Prelude/Common.hs

1317test/Streamly/Test/Prelude/Fold.hs

1418test/Streamly/Test/Prelude/Rate.hs

1519test/Streamly/Test/Prelude/Serial.hs

@@ -27,6 +31,7 @@ benchmark/Streamly/Benchmark/Data/Stream/StreamK.hs

2731benchmark/Streamly/Benchmark/Data/Unfold.hs

2832benchmark/Streamly/Benchmark/FileSystem/Handle.hs

2933benchmark/Streamly/Benchmark/Prelude/Async.hs

34+benchmark/Streamly/Benchmark/Prelude/Concurrent.hs

3035benchmark/Streamly/Benchmark/Prelude/Merge.hs

3136benchmark/Streamly/Benchmark/Prelude/Parallel.hs

3237benchmark/Streamly/Benchmark/Prelude/Rate.hs

benchmark/Streamly/Benchmark/Data/Array/Stream.hs+2−2

@@ -39,7 +39,7 @@ import Prelude hiding ()

3939

4040import qualified Streamly.Data.Stream as Stream

4141import qualified Streamly.Internal.Data.Array as Array

42−import qualified Streamly.Internal.Data.Stream.Chunked as ArrayStream

42+import qualified Streamly.Internal.Data.Array.Stream as ArrayStream

4343import qualified Streamly.Internal.Data.Fold as Fold

4444import qualified Streamly.Internal.Data.Parser as Parser

4545import qualified Streamly.Internal.Data.Stream as Stream

@@ -53,7 +53,7 @@ import Streamly.Benchmark.Common.Handle

5353import Control.Monad.IO.Class (MonadIO)

5454

5555#ifdef INSPECTION

56−import Streamly.Internal.Data.Unbox (Unbox)

56+import Streamly.Internal.Data.MutByteArray (Unbox)

5757import Streamly.Internal.Data.Stream (Step(..))

5858import Test.Inspection

5959#endif

benchmark/Streamly/Benchmark/Data/Fold.hs+2−2

@@ -269,13 +269,13 @@ unzip = Stream.fold $ FL.lmap (\a -> (a, a)) (FL.unzip FL.sum FL.length)

269269{-# INLINE unzipWithFstM #-}

270270unzipWithFstM :: Monad m => Stream m Int -> m (Int, Int)

271271unzipWithFstM = do

272− let f = \a -> return (a + 1, a)

272+ let f a = return (a + 1, a)

273273 Stream.fold (FL.unzipWithFstM f FL.sum FL.length)

274274

275275{-# INLINE unzipWithMinM #-}

276276unzipWithMinM :: Monad m => Stream m Int -> m (Int, Int)

277277unzipWithMinM = do

278− let f = \a -> return (a + 1, a)

278+ let f a = return (a + 1, a)

279279 Stream.fold (FL.unzipWithMinM f FL.sum FL.length)

280280

281281-------------------------------------------------------------------------------

benchmark/Streamly/Benchmark/Data/Serialize.hs+33−47

@@ -1,4 +1,5 @@

11{-# LANGUAGE TemplateHaskell #-}

2+{- HLINT ignore -}

34#undef FUSION_CHECK

45#ifdef FUSION_CHECK

@@ -18,28 +19,20 @@ module Main (main) where

1819-------------------------------------------------------------------------------

1920

2021import Control.DeepSeq (NFData(..), deepseq)

22+import Control.Monad (when, void)

2123import GHC.Generics (Generic)

2224import System.Random (randomRIO)

2325#ifndef USE_UNBOX

2426import Test.QuickCheck (Arbitrary, arbitrary)

2527#endif

2628

29+import Streamly.Internal.Data.MutByteArray

30+import qualified Streamly.Internal.Data.MutByteArray as MBA

2731#ifdef USE_UNBOX

2832import Data.Proxy (Proxy(..))

29−import Streamly.Internal.Data.Unbox

3033#else

3134import Control.DeepSeq (force)

3235import Test.QuickCheck (oneof, generate)

33−import Streamly.Internal.Data.Unbox (newBytes, MutableByteArray)

34−import Streamly.Internal.Data.Serialize hiding (encode)

35−#endif

36−

37−#ifdef USE_TH

38−#ifdef USE_UNBOX

39−import Streamly.Internal.Data.Unbox.TH

40−#else

41−import Streamly.Internal.Data.Serialize.TH

42−#endif

4336#endif

4437

4538import Test.Tasty.Bench

@@ -52,14 +45,14 @@ import Streamly.Benchmark.Data.Serialize.RecNonCompatible

5245

5346#ifdef USE_UNBOX

5447#define SERIALIZE_CLASS Unbox

55−#define DERIVE_CLASS deriveUnbox

56−#define SERIALIZE_OP pokeByteIndex

57−#define DESERIALIZE_OP peekByteIndex

48+#define DERIVE_CLASS(typ) $(deriveUnbox [d|instance Unbox typ|])

49+#define SERIALIZE_OP pokeAt

50+#define DESERIALIZE_OP peekAt

5851#else

5952#define SERIALIZE_CLASS Serialize

60−#define DERIVE_CLASS deriveSerialize

61−#define SERIALIZE_OP serialize

62−#define DESERIALIZE_OP deserialize

53+#define DERIVE_CLASS(typ) $(deriveSerialize [d|instance Serialize typ|])

54+#define SERIALIZE_OP serializeAt

55+#define DESERIALIZE_OP deserializeAt

6356#endif

6457

6558-------------------------------------------------------------------------------

@@ -72,7 +65,7 @@ data Unit = Unit

7265#ifndef USE_TH

7366instance SERIALIZE_CLASS Unit

7467#else

75−$(DERIVE_CLASS ''Unit)

68+DERIVE_CLASS(Unit)

7669#endif

7770

7871instance NFData Unit where

@@ -87,7 +80,7 @@ data Sum2

8780#ifndef USE_TH

8881instance SERIALIZE_CLASS Sum2

8982#else

90−$(DERIVE_CLASS ''Sum2)

83+DERIVE_CLASS(Sum2)

9184#endif

9285

9386instance NFData Sum2 where

@@ -125,7 +118,7 @@ data Sum25

125118#ifndef USE_TH

126119instance SERIALIZE_CLASS Sum25

127120#else

128−$(DERIVE_CLASS ''Sum25)

121+DERIVE_CLASS(Sum25)

129122#endif

130123

131124instance NFData Sum25 where

@@ -164,7 +157,7 @@ data Product25

164157#ifndef USE_TH

165158instance SERIALIZE_CLASS Product25

166159#else

167−$(DERIVE_CLASS ''Product25)

160+DERIVE_CLASS(Product25)

168161#endif

169162

170163instance NFData Product25 where

@@ -244,7 +237,7 @@ data CustomDT1

244237#ifndef USE_TH

245238instance SERIALIZE_CLASS CustomDT1

246239#else

247−$(DERIVE_CLASS ''CustomDT1)

240+DERIVE_CLASS(CustomDT1)

248241#endif

249242

250243instance NFData CustomDT1 where

@@ -267,19 +260,20 @@ data BinTree a

267260#ifndef USE_TH

268261instance Serialize (BinTree a)

269262#else

270−$(deriveSerialize ''BinTree)

263+$(deriveSerialize [d|instance Serialize a => Serialize (BinTree a)|])

271264#endif

272265

273266instance NFData a => NFData (BinTree a) where

274267 {-# INLINE rnf #-}

275268 rnf (Leaf a) = rnf a `seq` ()

276− rnf (Tree l r) = rnf l `seq` rnf r `seq` ()

269+ rnf (Tree l r) = rnf l `seq` rnf r

277270

271+-- XXX This may not terminate, or could become really large.

278272instance Arbitrary a => Arbitrary (BinTree a) where

279273 arbitrary = oneof [Leaf <$> arbitrary, Tree <$> arbitrary <*> arbitrary]

280274

281275mkBinTree :: (Arbitrary a) => Int -> IO (BinTree a)

282−mkBinTree = go (generate $ arbitrary)

276+mkBinTree = go (generate arbitrary)

283277

284278 where

285279

@@ -297,23 +291,21 @@ getSize :: forall a. SERIALIZE_CLASS a => a -> Int

297291#ifdef USE_UNBOX

298292getSize _ = sizeOf (Proxy :: Proxy a)

299293#else

300−getSize = size 0

294+getSize = addSizeTo 0

301295#endif

302296

303297-------------------------------------------------------------------------------

304298-- Common helpers

305299-------------------------------------------------------------------------------

306300

301+{- HLINT ignore "Eta reduce" -}

307302-- Parts of "f" that are dependent on val will not be optimized out.

308303{-# INLINE loop #-}

309304loop :: Int -> (a -> IO b) -> a -> IO ()

310305loop count f val = go count val

311306 where

312307

313− go n x = do

314− if n > 0

315− then f x >> go (n-1) x

316− else return ()

308+ go n x = when (n > 0) $ f x >> go (n-1) x

317309

318310-- The first arg of "f" is the environment which is not threaded around in the

319311-- loop.

@@ -322,10 +314,7 @@ loopWith :: Int -> (e -> a -> IO b) -> e -> a -> IO ()

322314loopWith count f e val = go count val

323315 where

324316

325− go n x = do

326− if n > 0

327− then f e x >> go (n-1) x

328− else return ()

317+ go n x = when (n > 0) $ f e x >> go (n-1) x

329318

330319benchSink :: NFData b => String -> Int -> (Int -> IO b) -> Benchmark

331320benchSink name times f = bench name (nfIO (randomRIO (times, times) >>= f))

@@ -336,7 +325,7 @@ benchSink name times f = bench name (nfIO (randomRIO (times, times) >>= f))

336325

337326{-

338327{-# INLINE pokeWithSize #-}

339−pokeWithSize :: SERIALIZE_CLASS a => MutableByteArray -> a -> IO ()

328+pokeWithSize :: SERIALIZE_CLASS a => MutByteArray -> a -> IO ()

340329pokeWithSize arr val = do

341330 let n = getSize val

342331 n `seq` SERIALIZE_OP 0 arr val >> return ()

@@ -345,27 +334,27 @@ pokeWithSize arr val = do

345334pokeTimesWithSize :: SERIALIZE_CLASS a => a -> Int -> IO ()

346335pokeTimesWithSize val times = do

347336 let n = getSize val

348− arr <- newBytes n

337+ arr <- MBA.new n

349338 loopWith times pokeWithSize arr val

350339-}

351340

352341{-# INLINE poke #-}

353−poke :: SERIALIZE_CLASS a => MutableByteArray -> a -> IO ()

354−poke arr val = SERIALIZE_OP 0 arr val >> return ()

342+poke :: SERIALIZE_CLASS a => MutByteArray -> a -> IO ()

343+poke arr val = void (SERIALIZE_OP 0 arr val)

355344

356345{-# INLINE pokeTimes #-}

357346pokeTimes :: SERIALIZE_CLASS a => a -> Int -> IO ()

358347pokeTimes val times = do

359348 let n = getSize val

360− arr <- newBytes n

349+ arr <- MBA.new n

361350 loopWith times poke arr val

362351

363352{-# INLINE encode #-}

364353encode :: SERIALIZE_CLASS a => a -> IO ()

365354encode val = do

366355 let n = getSize val

367− arr <- newBytes n

368− SERIALIZE_OP 0 arr val >> return ()

356+ arr <- MBA.new n

357+ void (SERIALIZE_OP 0 arr val)

369358

370359{-# INLINE encodeTimes #-}

371360encodeTimes :: SERIALIZE_CLASS a => a -> Int -> IO ()

@@ -373,7 +362,7 @@ encodeTimes val times = loop times encode val

373362

374363{-# INLINE peek #-}

375364peek :: forall a. (NFData a, SERIALIZE_CLASS a) =>

376− (a, Int) -> MutableByteArray -> IO ()

365+ (a, Int) -> MutByteArray -> IO ()

377366#ifdef USE_UNBOX

378367peek (_val, _) arr = do

379368 (val1 :: a) <- DESERIALIZE_OP 0 arr

@@ -400,7 +389,7 @@ peek (_val, n) arr = do

400389{-# INLINE peekTimes #-}

401390peekTimes :: (NFData a, SERIALIZE_CLASS a) => Int -> a -> Int -> IO ()

402391peekTimes n val times = do

403− arr <- newBytes n

392+ arr <- MBA.new n

404393 _ <- SERIALIZE_OP 0 arr val

405394 loopWith times peek (val, n) arr

406395

@@ -408,7 +397,7 @@ peekTimes n val times = do

408397trip :: forall a. (NFData a, SERIALIZE_CLASS a) => a -> IO ()

409398trip val = do

410399 let n = getSize val

411− arr <- newBytes n

400+ arr <- MBA.new n

412401 _ <- SERIALIZE_OP 0 arr val

413402#ifdef USE_UNBOX

414403 (val1 :: a) <- DESERIALIZE_OP 0 arr

@@ -571,8 +560,6 @@ main = do

571560 let !len = length lInt -- evaluate the list

572561#endif

573562#ifndef FUSION_CHECK

574− -- This can take too much memory/CPU, need to restrict the test

575− -- runQC

576563#ifdef USE_UNBOX

577564 runWithCLIOpts defaultStreamSize allBenchmarks

578565#else

@@ -580,7 +567,6 @@ main = do

580567 `seq` runWithCLIOpts

581568 defaultStreamSize

582569 (allBenchmarks tInt lInt recL recR)

583−

584570#endif

585571#else

586572 -- Enable FUSION_CHECK macro at the beginning of the file

benchmark/Streamly/Benchmark/Data/Serialize/RecCompatible.hs+3−3

@@ -9,10 +9,10 @@ module Streamly.Benchmark.Data.Serialize.RecCompatible

99-- Imports

1010--------------------------------------------------------------------------------

1111

12−import Streamly.Internal.Data.Serialize (Serialize)

12+import Streamly.Internal.Data.MutByteArray (Serialize)

1313import Streamly.Benchmark.Data.Serialize.TH (genLargeRecord)

1414

15−import qualified Streamly.Internal.Data.Serialize.TH as Serialize

15+import qualified Streamly.Internal.Data.MutByteArray as Serialize

1616

1717--------------------------------------------------------------------------------

1818-- Code

@@ -20,5 +20,5 @@ import qualified Streamly.Internal.Data.Serialize.TH as Serialize

2020

2121$(genLargeRecord "RecCompatible" 50)

2222$(Serialize.deriveSerializeWith

23− (Serialize.defaultConfig {Serialize.recordSyntaxWithHeader = True})

23+ (Serialize.encodeRecordFields True)

2424 [d|instance Serialize RecCompatible|])

benchmark/Streamly/Benchmark/Data/Serialize/RecNonCompatible.hs+2−2

@@ -11,11 +11,11 @@ module Streamly.Benchmark.Data.Serialize.RecNonCompatible

1111

1212import Streamly.Benchmark.Data.Serialize.TH (genLargeRecord)

1313

14−import qualified Streamly.Internal.Data.Serialize.TH as Serialize

14+import qualified Streamly.Internal.Data.MutByteArray as Serialize

1515

1616--------------------------------------------------------------------------------

1717-- Code

1818--------------------------------------------------------------------------------

1919

2020$(genLargeRecord "RecNonCompatible" 50)

21−$(Serialize.deriveSerialize ''RecNonCompatible)

21+$(Serialize.deriveSerialize [d|instance Serialize.Serialize RecNonCompatible|])

benchmark/Streamly/Benchmark/Data/Serialize/TH.hs+4−4

@@ -8,7 +8,7 @@ module Streamly.Benchmark.Data.Serialize.TH (genLargeRecord) where

99import Language.Haskell.TH

1010

11−import Streamly.Internal.Data.Serialize.TH.Bottom (makeI)

11+import Streamly.Internal.Data.MutByteArray (makeI)

1212import Control.DeepSeq (NFData(..))

1313

1414--------------------------------------------------------------------------------

@@ -18,7 +18,7 @@ import Control.DeepSeq (NFData(..))

1818genLargeRecord :: String -> Int -> Q [Dec]

1919genLargeRecord tyName numFields =

2020 sequence

21− ([ dataD

21+ [ dataD

2222 (pure [])

2323 (mkName tyName)

2424 []

@@ -28,7 +28,7 @@ genLargeRecord tyName numFields =

2828 , mkValueSigDec

2929 , mkValueDec

3030 , nfDataInstance tyName

31− ])

31+ ]

3232

3333 where

3434

@@ -64,7 +64,7 @@ genLargeRecord tyName numFields =

6464 (foldl

6565 (\b a -> [|$(b) $(a)|])

6666 (conE (mkName tyName))

67− (const (conE '()) <$> [0 .. (numFields - 1)])))

67+ (conE '() <$ [0 .. (numFields - 1)])))

6868 []

6969 ]

7070 mkCon nm = recC (mkName nm) (mkField <$> [0 .. (numFields - 1)])

benchmark/Streamly/Benchmark/Data/Stream/ConcurrentCommon.hs+2−2

@@ -14,12 +14,12 @@ where

1414import Stream.Common

1515 (composeN, benchIO, benchIOSink, benchIOSrc, sourceUnfoldrM)

1616import Streamly.Data.Stream (Stream)

17−import Streamly.Internal.Data.Stream.Concurrent (MonadAsync, Config)

17+import Streamly.Internal.Data.Stream.Prelude (MonadAsync, Config)

1818

1919import qualified Data.List as List

2020import qualified Streamly.Data.Fold as Fold

2121import qualified Streamly.Data.Stream as Stream

22−import qualified Streamly.Internal.Data.Stream.Concurrent as Async

22+import qualified Streamly.Internal.Data.Stream.Prelude as Async

2323

2424import Test.Tasty.Bench

2525import Prelude hiding (mapM)

benchmark/Streamly/Benchmark/Data/Stream/ConcurrentEager.hs+1−1

@@ -9,7 +9,7 @@

99import Stream.ConcurrentCommon (allBenchmarks)

1010import Streamly.Benchmark.Common (runWithCLIOpts, defaultStreamSize)

1111

12−import qualified Streamly.Internal.Data.Stream.Concurrent as Async

12+import qualified Streamly.Internal.Data.Stream.Prelude as Async

1313

1414moduleName :: String

1515moduleName = "Data.Stream.ConcurrentEager"

benchmark/Streamly/Benchmark/Data/Stream/ConcurrentInterleaved.hs+1−1

@@ -9,7 +9,7 @@

99import Stream.ConcurrentCommon (allBenchmarks)

1010import Streamly.Benchmark.Common (runWithCLIOpts, defaultStreamSize)

1111

12−import qualified Streamly.Internal.Data.Stream.Concurrent as Async

12+import qualified Streamly.Internal.Data.Stream.Prelude as Async

1313

1414moduleName :: String

1515moduleName = "Data.Stream.ConcurrentInterleaved"

benchmark/Streamly/Benchmark/Data/Stream/ConcurrentOrdered.hs+1−1

@@ -9,7 +9,7 @@

99import Stream.ConcurrentCommon (allBenchmarks)

1010import Streamly.Benchmark.Common (runWithCLIOpts, defaultStreamSize)

1111

12−import qualified Streamly.Internal.Data.Stream.Concurrent as Async

12+import qualified Streamly.Internal.Data.Stream.Prelude as Async

1313

1414moduleName :: String

1515moduleName = "Data.Stream.ConcurrentOrdered"

benchmark/Streamly/Benchmark/Data/Stream/Eliminate.hs+2

@@ -132,12 +132,14 @@ foldableAny :: Int -> Int -> Bool

132132foldableAny value n =

133133 Prelude.any (> (value + 1)) (sourceUnfoldr value n :: Stream Identity Int)

134134

135+{- HLINT ignore "Use all"-}

135136{-# INLINE foldableAnd #-}

136137foldableAnd :: Int -> Int -> Bool

137138foldableAnd value n =

138139 Prelude.and $ fmap

139140 (<= (value + 1)) (sourceUnfoldr value n :: Stream Identity Int)

140141

142+{- HLINT ignore "Use any"-}

141143{-# INLINE foldableOr #-}

142144foldableOr :: Int -> Int -> Bool

143145foldableOr value n =

benchmark/Streamly/Benchmark/Data/Stream/Exceptions.hs+2−2

@@ -36,13 +36,13 @@ import System.IO (Handle, hClose, hPutChar)

3636import qualified Streamly.FileSystem.Handle as FH

3737import qualified Streamly.Internal.FileSystem.Handle as IFH

3838import qualified Streamly.Internal.Data.Unfold as IUF

39−import qualified Streamly.Internal.Data.Unfold.Exception as IUF

39+import qualified Streamly.Internal.Data.Unfold.Prelude as IUF

4040

4141#ifdef USE_PRELUDE

4242import qualified Streamly.Internal.Data.Stream.IsStream as Stream

4343#else

4444import qualified Streamly.Internal.Data.Stream as Stream

45−import qualified Streamly.Internal.Data.Stream.Exception.Lifted as Stream

45+import qualified Streamly.Internal.Data.Stream.Prelude as Stream

4646#endif

4747

4848import Test.Tasty.Bench hiding (env)

benchmark/Streamly/Benchmark/Data/Stream/Reduce.hs+1−1

@@ -48,7 +48,7 @@ import Streamly.Internal.Data.Stream (Stream)

4848import qualified Streamly.Internal.Data.Stream as S

4949#ifndef USE_STREAMLY_CORE

5050import qualified Streamly.Data.Stream.Prelude as S

51−import qualified Streamly.Internal.Data.Stream.Time as S

51+import qualified Streamly.Internal.Data.Stream.Prelude as S

5252#endif

5353

5454#ifdef USE_STREAMK

benchmark/Streamly/Benchmark/Data/Stream/StreamKAlt.hs+1−1

@@ -452,4 +452,4 @@ o_1_space =

452452 ]

453453

454454main :: IO ()

455−main = defaultMain $ concat [o_1_space]

455+main = defaultMain [o_1_space]

benchmark/Streamly/Benchmark/Data/Stream/ToStreamK.hs+1−1

@@ -120,7 +120,7 @@ fromFoldable streamLen n = S.fromFoldable [n..n+streamLen]

120120{-# INLINE fromFoldableM #-}

121121fromFoldableM :: Monad m => Int -> Int -> Stream m Int

122122fromFoldableM streamLen n =

123− Prelude.foldr S.consM S.nil (Prelude.fmap return [n..n+streamLen])

123+ Prelude.foldr (S.consM . return) S.nil [n .. n + streamLen]

124124

125125{-

126126{-# INLINABLE concatMapFoldableWith #-}

benchmark/Streamly/Benchmark/Data/Stream/Transform.hs+1−1

@@ -46,7 +46,7 @@ import Streamly.Internal.Data.Time.Units

4646import Streamly.Internal.Data.Stream (Stream)

4747import qualified Streamly.Internal.Data.Stream as Stream

4848#ifndef USE_STREAMLY_CORE

49−import qualified Streamly.Internal.Data.Stream.Time as Stream

49+import qualified Streamly.Internal.Data.Stream.Prelude as Stream

5050#endif

5151#ifdef USE_STREAMK

5252import Control.DeepSeq (NFData(..))

benchmark/Streamly/Benchmark/Data/Unfold.hs+1−1

@@ -31,7 +31,7 @@ import qualified Prelude

3131import qualified Streamly.FileSystem.Handle as FH

3232import qualified Streamly.Internal.Data.Fold as FL

3333import qualified Streamly.Internal.Data.Unfold as UF

34−import qualified Streamly.Internal.Data.Unfold.Exception as UF

34+import qualified Streamly.Internal.Data.Unfold.Prelude as UF

3535import qualified Streamly.Internal.Data.Stream as S

3636import qualified Streamly.Internal.Data.Stream as D

3737import qualified Streamly.Internal.Data.StreamK as K

benchmark/Streamly/Benchmark/FileSystem/Handle/Read.hs+7−8

@@ -33,7 +33,6 @@ import qualified Streamly.Data.Stream as Stream

3333import qualified Streamly.Data.Fold as Fold

3434import qualified Streamly.FileSystem.Handle as FH

3535import qualified Streamly.Internal.Data.Array as A

36−import qualified Streamly.Internal.Data.Array as AT

3736import qualified Streamly.Internal.Data.Fold as FL

3837import qualified Streamly.Internal.Data.Stream as IP

3938import qualified Streamly.Internal.FileSystem.Handle as IFH

@@ -48,7 +47,7 @@ import Streamly.Benchmark.Common.Handle

4847#ifdef INSPECTION

4948import Streamly.Internal.Data.Stream (Step(..), FoldMany)

5049

51−import qualified Streamly.Internal.Data.MutArray as MA

50+import qualified Streamly.Internal.Data.MutArray as MutArray

5251import qualified Streamly.Internal.Data.Stream as D

5352import qualified Streamly.Internal.Data.Unfold as IUF

5453

@@ -69,7 +68,7 @@ readLast = S.fold Fold.last . S.unfold FH.reader

6968inspect $ hasNoTypeClasses 'readLast

7069inspect $ 'readLast `hasNoType` ''Step -- S.unfold

7170inspect $ 'readLast `hasNoType` ''IUF.ConcatState -- FH.read/UF.many

72−inspect $ 'readLast `hasNoType` ''MA.ArrayUnsafe -- FH.read/A.read

71+inspect $ 'readLast `hasNoType` ''MutArray.ArrayUnsafe -- FH.read/A.read

7372#endif

7473

7574-- assert that flattenArrays constructors are not present

@@ -81,7 +80,7 @@ readCountBytes = S.fold Fold.length . S.unfold FH.reader

8180inspect $ hasNoTypeClasses 'readCountBytes

8281inspect $ 'readCountBytes `hasNoType` ''Step -- S.unfold

8382inspect $ 'readCountBytes `hasNoType` ''IUF.ConcatState -- FH.read/UF.many

84−inspect $ 'readCountBytes `hasNoType` ''MA.ArrayUnsafe -- FH.read/A.read

83+inspect $ 'readCountBytes `hasNoType` ''MutArray.ArrayUnsafe -- FH.read/A.read

8584#endif

8685

8786-- | Count the number of lines in a file.

@@ -96,7 +95,7 @@ readCountLines =

9695inspect $ hasNoTypeClasses 'readCountLines

9796inspect $ 'readCountLines `hasNoType` ''Step

9897inspect $ 'readCountLines `hasNoType` ''IUF.ConcatState -- FH.read/UF.many

99−inspect $ 'readCountLines `hasNoType` ''MA.ArrayUnsafe -- FH.read/A.read

98+inspect $ 'readCountLines `hasNoType` ''MutArray.ArrayUnsafe -- FH.read/A.read

10099#endif

101100

102101-- | Count the number of words in a file.

@@ -120,7 +119,7 @@ readSumBytes = S.fold Fold.sum . S.unfold FH.reader

120119inspect $ hasNoTypeClasses 'readSumBytes

121120inspect $ 'readSumBytes `hasNoType` ''Step

122121inspect $ 'readSumBytes `hasNoType` ''IUF.ConcatState -- FH.read/UF.many

123−inspect $ 'readSumBytes `hasNoType` ''MA.ArrayUnsafe -- FH.read/A.read

122+inspect $ 'readSumBytes `hasNoType` ''MutArray.ArrayUnsafe -- FH.read/A.read

124123#endif

125124

126125-- XXX When we mark this with INLINE and we have two benchmarks using S.drain

@@ -236,13 +235,13 @@ groupsOf :: Int -> Handle -> IO Int

236235groupsOf n inh =

237236 -- writeNUnsafe gives 2.5x boost here over writeN.

238237 S.fold Fold.length

239− $ IP.groupsOf n (AT.writeNUnsafe n) (S.unfold FH.reader inh)

238+ $ IP.groupsOf n (A.writeNUnsafe n) (S.unfold FH.reader inh)

240239

241240#ifdef INSPECTION

242241inspect $ hasNoTypeClasses 'groupsOf

243242inspect $ 'groupsOf `hasNoType` ''Step

244243inspect $ 'groupsOf `hasNoType` ''FoldMany

245−inspect $ 'groupsOf `hasNoType` ''AT.ArrayUnsafe -- AT.writeNUnsafe

244+inspect $ 'groupsOf `hasNoType` ''MutArray.ArrayUnsafe -- AT.writeNUnsafe

246245 -- FH.read/A.read

247246inspect $ 'groupsOf `hasNoType` ''IUF.ConcatState -- FH.read/UF.many

248247#endif

benchmark/Streamly/Benchmark/FileSystem/Handle/ReadWrite.hs+8−9

@@ -42,8 +42,7 @@ import Streamly.Internal.Data.Stream (Step(..))

4242

4343import qualified Streamly.Internal.Data.Stream as D

4444import qualified Streamly.Internal.Data.Tuple.Strict as Strict

45−import qualified Streamly.Internal.Data.Stream.MutChunked as MAS

46−import qualified Streamly.Internal.Data.Array as AT

45+import qualified Streamly.Internal.Data.MutArray as MutArray

4746

4847import Test.Inspection

4948#endif

@@ -83,7 +82,7 @@ copyStream inh outh = S.fold (FH.write outh) (S.unfold FH.reader inh)

8382inspect $ hasNoTypeClasses 'copyStream

8483inspect $ 'copyStream `hasNoType` ''Step -- S.unfold

8584inspect $ 'copyStream `hasNoType` ''IUF.ConcatState -- FH.read/UF.many

86−inspect $ 'copyStream `hasNoType` ''AT.ArrayUnsafe -- FH.write/writeNUnsafe

85+inspect $ 'copyStream `hasNoType` ''MutArray.ArrayUnsafe -- FH.write/writeNUnsafe

8786 -- FH.read/A.read

8887inspect $ 'copyStream `hasNoType` ''Strict.Tuple3' -- FH.write/chunksOf

8988#endif

@@ -109,8 +108,8 @@ readFromBytesNull inh devNull = IFH.putBytes devNull $ S.unfold FH.reader inh

109108#ifdef INSPECTION

110109inspect $ hasNoTypeClasses 'readFromBytesNull

111110inspect $ 'readFromBytesNull `hasNoType` ''Step

112−inspect $ 'readFromBytesNull `hasNoType` ''MAS.SpliceState

113−inspect $ 'readFromBytesNull `hasNoType` ''AT.ArrayUnsafe -- FH.fromBytes/S.chunksOf

111+inspect $ 'readFromBytesNull `hasNoType` ''MutArray.SpliceState

112+inspect $ 'readFromBytesNull `hasNoType` ''MutArray.ArrayUnsafe -- FH.fromBytes/S.chunksOf

114113inspect $ 'readFromBytesNull `hasNoType` ''D.FoldMany

115114#endif

116115

@@ -123,8 +122,8 @@ readWithFromBytesNull inh devNull =

123122#ifdef INSPECTION

124123inspect $ hasNoTypeClasses 'readWithFromBytesNull

125124inspect $ 'readWithFromBytesNull `hasNoType` ''Step

126−inspect $ 'readWithFromBytesNull `hasNoType` ''MAS.SpliceState

127−inspect $ 'readWithFromBytesNull `hasNoType` ''AT.ArrayUnsafe -- FH.fromBytes/S.chunksOf

125+inspect $ 'readWithFromBytesNull `hasNoType` ''MutArray.SpliceState

126+inspect $ 'readWithFromBytesNull `hasNoType` ''MutArray.ArrayUnsafe -- FH.fromBytes/S.chunksOf

128127inspect $ 'readWithFromBytesNull `hasNoType` ''D.FoldMany

129128#endif

130129

@@ -171,7 +170,7 @@ writeReadWith inh devNull = IUF.fold fld unf (defaultChunkSize, inh)

171170inspect $ hasNoTypeClasses 'writeReadWith

172171inspect $ 'writeReadWith `hasNoType` ''Step

173172inspect $ 'writeReadWith `hasNoType` ''IUF.ConcatState -- FH.read/UF.many

174−inspect $ 'writeReadWith `hasNoType` ''AT.ArrayUnsafe -- FH.write/writeNUnsafe

173+inspect $ 'writeReadWith `hasNoType` ''MutArray.ArrayUnsafe -- FH.write/writeNUnsafe

175174 -- FH.read/A.read

176175#endif

177176

@@ -188,7 +187,7 @@ writeRead inh devNull = IUF.fold fld unf inh

188187inspect $ hasNoTypeClasses 'writeRead

189188inspect $ 'writeRead `hasNoType` ''Step

190189inspect $ 'writeRead `hasNoType` ''IUF.ConcatState -- FH.read/UF.many

191−inspect $ 'writeRead `hasNoType` ''AT.ArrayUnsafe -- FH.write/writeNUnsafe

190+inspect $ 'writeRead `hasNoType` ''MutArray.ArrayUnsafe -- FH.write/writeNUnsafe

192191 -- FH.read/A.read

193192#endif

194193

benchmark/Streamly/Benchmark/Unicode/Stream.hs+5−4

@@ -43,11 +43,12 @@ import Streamly.Benchmark.Common

4343import Streamly.Benchmark.Common.Handle

4444

4545#ifdef INSPECTION

46−import Streamly.Internal.Data.Unbox (Unbox)

46+import Streamly.Internal.Data.MutByteArray (Unbox)

4747import Streamly.Internal.Data.Stream (Step(..))

48+import qualified Streamly.Internal.Data.Array as Array

49+import qualified Streamly.Internal.Data.MutArray as MutArray

4850import qualified Streamly.Internal.Data.Fold as Fold

4951import qualified Streamly.Internal.Data.Tuple.Strict as Strict

50−import qualified Streamly.Internal.Data.Array as Array

5152

5253import Test.Inspection

5354#endif

@@ -223,7 +224,7 @@ inspect $ 'copyStreamLatin1' `hasNoType` ''Step

223224inspect $ 'copyStreamLatin1' `hasNoType` ''Unfold.ConcatState -- Handle.read/UF.many

224225

225226inspect $ 'copyStreamLatin1' `hasNoType` ''Fold.Step

226−inspect $ 'copyStreamLatin1' `hasNoType` ''Array.ArrayUnsafe -- Handle.write/writeNUnsafe

227+inspect $ 'copyStreamLatin1' `hasNoType` ''MutArray.ArrayUnsafe -- Handle.write/writeNUnsafe

227228 -- Handle.read/Array.read

228229inspect $ 'copyStreamLatin1' `hasNoType` ''Strict.Tuple3' -- Handle.write/chunksOf

229230#endif

@@ -244,7 +245,7 @@ inspect $ 'copyStreamLatin1 `hasNoType` ''Unfold.ConcatState -- Handle.read/UF.m

244245

245246inspect $ 'copyStreamLatin1 `hasNoType` ''Fold.ManyState

246247inspect $ 'copyStreamLatin1 `hasNoType` ''Fold.Step

247−inspect $ 'copyStreamLatin1 `hasNoType` ''Array.ArrayUnsafe -- Handle.write/writeNUnsafe

248+inspect $ 'copyStreamLatin1 `hasNoType` ''MutArray.ArrayUnsafe -- Handle.write/writeNUnsafe

248249 -- Handle.read/Array.read

249250inspect $ 'copyStreamLatin1 `hasNoType` ''Strict.Tuple3' -- Handle.write/chunksOf

250251#endif

benchmark/streamly-benchmarks.cabal+1−1

@@ -188,7 +188,7 @@ common bench-depends

188188 , unordered-containers >= 0.2 && < 0.3

189189 , process >= 1.4 && < 1.7

190190 , directory >= 1.2.2 && < 1.4

191− , filepath >= 1.4.1 && < 1.5

191+ , filepath >= 1.4.1 && < 1.6

192192 , ghc-prim >= 0.4 && < 0.12

193193 , tasty-bench >= 0.3 && < 0.4

194194 , tasty >= 1.4.1 && < 1.6

configure+9−9

@@ -1,6 +1,6 @@

11#! /bin/sh

22# Guess values for system-dependent variables and create Makefiles.

3−# Generated by GNU Autoconf 2.71 for streamly 0.10.0.

3+# Generated by GNU Autoconf 2.71 for streamly 0.10.1.

44#

55# Report bugs to <streamly@composewell.com>.

66#

@@ -610,8 +610,8 @@ MAKEFLAGS=

610610# Identity of this package.

611611PACKAGE_NAME='streamly'

612612PACKAGE_TARNAME='streamly'

613−PACKAGE_VERSION='0.10.0'

614−PACKAGE_STRING='streamly 0.10.0'

613+PACKAGE_VERSION='0.10.1'

614+PACKAGE_STRING='streamly 0.10.1'

615615PACKAGE_BUGREPORT='streamly@composewell.com'

616616PACKAGE_URL='https://streamly.composewell.com'

617617

@@ -1224,7 +1224,7 @@ if test "$ac_init_help" = "long"; then

12241224 # Omit some internal or obsolete options to make the list less imposing.

12251225 # This message is too long to be a string in the A/UX 3.1 sh.

12261226 cat <<_ACEOF

1227−\`configure' configures streamly 0.10.0 to adapt to many kinds of systems.

1227+\`configure' configures streamly 0.10.1 to adapt to many kinds of systems.

12281228

12291229Usage: $0 [OPTION]... [VAR=VALUE]...

12301230

@@ -1286,7 +1286,7 @@ fi

12861286

12871287if test -n "$ac_init_help"; then

12881288 case $ac_init_help in

1289− short | recursive ) echo "Configuration of streamly 0.10.0:";;

1289+ short | recursive ) echo "Configuration of streamly 0.10.1:";;

12901290 esac

12911291 cat <<\_ACEOF

12921292

@@ -1372,7 +1372,7 @@ fi

13721372test -n "$ac_init_help" && exit $ac_status

13731373if $ac_init_version; then

13741374 cat <<\_ACEOF

1375−streamly configure 0.10.0

1375+streamly configure 0.10.1

13761376generated by GNU Autoconf 2.71

13771377

@@ -1500,7 +1500,7 @@ cat >config.log <<_ACEOF

15001500This file contains any messages produced by compilers while

15011501running configure, to aid debugging if configure makes a mistake.

15021502

1503−It was created by streamly $as_me 0.10.0, which was

1503+It was created by streamly $as_me 0.10.1, which was

15041504generated by GNU Autoconf 2.71. Invocation command line was

15051505

15061506 $ $0$ac_configure_args_raw

@@ -3800,7 +3800,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1

38003800# report actual input values of CONFIG_FILES etc. instead of their

38013801# values after options handling.

38023802ac_log="

3803−This file was extended by streamly $as_me 0.10.0, which was

3803+This file was extended by streamly $as_me 0.10.1, which was

38043804generated by GNU Autoconf 2.71. Invocation command line was

38053805

38063806 CONFIG_FILES = $CONFIG_FILES

@@ -3856,7 +3856,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\

38563856cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1

38573857ac_cs_config='$ac_cs_config_escaped'

38583858ac_cs_version="\\

3859−streamly config.status 0.10.0

3859+streamly config.status 0.10.1

38603860configured by $0, generated by GNU Autoconf 2.71,

38613861 with options \\"\$ac_cs_config\\"

38623862

configure.ac+1−1

@@ -3,7 +3,7 @@

33# See https://www.gnu.org/software/autoconf/manual/autoconf.html for help on

44# the macros used in this file.

6−AC_INIT([streamly], [0.10.0], [streamly@composewell.com], [streamly], [https://streamly.composewell.com])

6+AC_INIT([streamly], [0.10.1], [streamly@composewell.com], [streamly], [https://streamly.composewell.com])

88# To suppress "WARNING: unrecognized options: --with-compiler"

99AC_ARG_WITH([compiler], [GHC])

core/configure+9−9

@@ -1,6 +1,6 @@

11#! /bin/sh

22# Guess values for system-dependent variables and create Makefiles.

3−# Generated by GNU Autoconf 2.71 for streamly-core 0.2.0.

3+# Generated by GNU Autoconf 2.71 for streamly-core 0.2.2.

44#

55# Report bugs to <streamly@composewell.com>.

66#

@@ -610,8 +610,8 @@ MAKEFLAGS=

610610# Identity of this package.

611611PACKAGE_NAME='streamly-core'

612612PACKAGE_TARNAME='streamly-core'

613−PACKAGE_VERSION='0.2.0'

614−PACKAGE_STRING='streamly-core 0.2.0'

613+PACKAGE_VERSION='0.2.2'

614+PACKAGE_STRING='streamly-core 0.2.2'

615615PACKAGE_BUGREPORT='streamly@composewell.com'

616616PACKAGE_URL='https://streamly.composewell.com'

617617

@@ -1256,7 +1256,7 @@ if test "$ac_init_help" = "long"; then

12561256 # Omit some internal or obsolete options to make the list less imposing.

12571257 # This message is too long to be a string in the A/UX 3.1 sh.

12581258 cat <<_ACEOF

1259−\`configure' configures streamly-core 0.2.0 to adapt to many kinds of systems.

1259+\`configure' configures streamly-core 0.2.2 to adapt to many kinds of systems.

12601260

12611261Usage: $0 [OPTION]... [VAR=VALUE]...

12621262

@@ -1318,7 +1318,7 @@ fi

13181318

13191319if test -n "$ac_init_help"; then

13201320 case $ac_init_help in

1321− short | recursive ) echo "Configuration of streamly-core 0.2.0:";;

1321+ short | recursive ) echo "Configuration of streamly-core 0.2.2:";;

13221322 esac

13231323 cat <<\_ACEOF

13241324

@@ -1404,7 +1404,7 @@ fi

14041404test -n "$ac_init_help" && exit $ac_status

14051405if $ac_init_version; then

14061406 cat <<\_ACEOF

1407−streamly-core configure 0.2.0

1407+streamly-core configure 0.2.2

14081408generated by GNU Autoconf 2.71

14091409

@@ -1622,7 +1622,7 @@ cat >config.log <<_ACEOF

16221622This file contains any messages produced by compilers while

16231623running configure, to aid debugging if configure makes a mistake.

16241624

1625−It was created by streamly-core $as_me 0.2.0, which was

1625+It was created by streamly-core $as_me 0.2.2, which was

16261626generated by GNU Autoconf 2.71. Invocation command line was

16271627

16281628 $ $0$ac_configure_args_raw

@@ -3832,7 +3832,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1

38323832# report actual input values of CONFIG_FILES etc. instead of their

38333833# values after options handling.

38343834ac_log="

3835−This file was extended by streamly-core $as_me 0.2.0, which was

3835+This file was extended by streamly-core $as_me 0.2.2, which was

38363836generated by GNU Autoconf 2.71. Invocation command line was

38373837

38383838 CONFIG_FILES = $CONFIG_FILES

@@ -3888,7 +3888,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\

38883888cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1

38893889ac_cs_config='$ac_cs_config_escaped'

38903890ac_cs_version="\\

3891−streamly-core config.status 0.2.0

3891+streamly-core config.status 0.2.2

38923892configured by $0, generated by GNU Autoconf 2.71,

38933893 with options \\"\$ac_cs_config\\"

38943894

core/configure.ac+1−1

@@ -3,7 +3,7 @@

33# See https://www.gnu.org/software/autoconf/manual/autoconf.html for help on

44# the macros used in this file.

6−AC_INIT([streamly-core], [0.2.0], [streamly@composewell.com], [streamly-core], [https://streamly.composewell.com])

6+AC_INIT([streamly-core], [0.2.2], [streamly@composewell.com], [streamly-core], [https://streamly.composewell.com])

88# To suppress "WARNING: unrecognized options: --with-compiler"

99AC_ARG_WITH([compiler], [GHC])

core/docs/ApiChangelogs/0.1.0-0.2.0.txtnew file+1471

@@ -0,0 +1,1471 @@

1+---------------------------------

2+API Annotations

3+---------------------------------

5+[A] : Added

6+[R] : Removed

7+[C] : Changed

8+[O] : Old definition

9+[N] : New definition

10+[D] : Deprecated

11+

12+---------------------------------

13+API diff

14+---------------------------------

15+

16+[C] Streamly.Unicode.Parser

17+ [A] double :: Monad m => Parser Char m Double

18+[C] Streamly.FileSystem.Handle

19+ [A] readWith :: MonadIO m => Int -> Handle -> Stream m Word8

20+ [A] readChunksWith :: MonadIO m => Int -> Handle -> Stream m (Array Word8)

21+[C] Streamly.FileSystem.Dir

22+ [C] readEither

23+ [O] readEither :: MonadIO m => FilePath -> Stream m (Either FilePath FilePath)

24+ [N] readEither :: (MonadIO m, MonadCatch m) => FilePath -> Stream m (Either FilePath FilePath)

25+ [C] read

26+ [O] read :: MonadIO m => FilePath -> Stream m FilePath

27+ [N] read :: (MonadIO m, MonadCatch m) => FilePath -> Stream m FilePath

28+[C] Streamly.Data.Unfold

29+ [A] class Enum a => Enumerable a

30+ [A] second :: b -> Unfold m (a, b) c -> Unfold m a c

31+ [A] first :: a -> Unfold m (a, b) c -> Unfold m b c

32+ [A] enumerateFromTo :: (Enumerable a, Monad m) => Unfold m (a, a) a

33+ [A] enumerateFromThenTo :: (Enumerable a, Monad m) => Unfold m (a, a, a) a

34+ [A] enumerateFromThen :: (Enumerable a, Monad m) => Unfold m (a, a) a

35+ [A] enumerateFrom :: (Enumerable a, Monad m) => Unfold m a a

36+[C] Streamly.Data.StreamK

37+ [C] parseChunks

38+ [O] parseChunks :: (Monad m, Unbox a) => ParserK a m b -> StreamK m (Array a) -> m (Either ParseError b)

39+ [N] parseChunks :: (Monad m, Unbox a) => ParserK (Array a) m b -> StreamK m (Array a) -> m (Either ParseError b)

40+ [C] parseBreakChunks

41+ [O] parseBreakChunks :: (Monad m, Unbox a) => ParserK a m b -> StreamK m (Array a) -> m (Either ParseError b, StreamK m (Array a))

42+ [N] parseBreakChunks :: (Monad m, Unbox a) => ParserK (Array a) m b -> StreamK m (Array a) -> m (Either ParseError b, StreamK m (Array a))

43+ [A] parseBreak :: forall m a b. Monad m => ParserK a m b -> StreamK m a -> m (Either ParseError b, StreamK m a)

44+ [A] parse :: Monad m => ParserK a m b -> StreamK m a -> m (Either ParseError b)

45+ [A] handle :: (MonadCatch m, Exception e) => (e -> m (StreamK m a)) -> StreamK m a -> StreamK m a

46+ [A] bracketIO :: (MonadIO m, MonadCatch m) => IO b -> (b -> IO c) -> (b -> StreamK m a) -> StreamK m a

47+[C] Streamly.Data.Stream

48+ [A] wordsBy :: Monad m => (a -> Bool) -> Fold m a b -> Stream m a -> Stream m b

49+ [A] splitOn :: Monad m => (a -> Bool) -> Fold m a b -> Stream m a -> Stream m b

50+ [C] handle

51+ [O] handle :: (MonadCatch m, Exception e) => (e -> Stream m a) -> Stream m a -> Stream m a

52+ [N] handle :: (MonadCatch m, Exception e) => (e -> m (Stream m a)) -> Stream m a -> Stream m a

53+ [A] groupsOf :: Monad m => Int -> Fold m a b -> Stream m a -> Stream m b

54+[C] Streamly.Data.ParserK

55+ [D] fromParser :: (MonadIO m, Unbox a) => Parser a m b -> ParserK (Array a) m b

56+ [D] fromFold :: (MonadIO m, Unbox a) => Fold m a b -> ParserK (Array a) m b

57+ [A] adaptCG :: Monad m => Parser a m b -> ParserK (Array a) m b

58+ [A] adaptC :: (Monad m, Unbox a) => Parser a m b -> ParserK (Array a) m b

59+ [A] adapt :: Monad m => Parser a m b -> ParserK a m b

60+[C] Streamly.Data.Parser

61+ [A] groupByRollingEither :: Monad m => (a -> a -> Bool) -> Fold m a b -> Fold m a c -> Parser a m (Either b c)

62+ [A] groupByRolling :: Monad m => (a -> a -> Bool) -> Fold m a b -> Parser a m b

63+[A] Streamly.Data.MutByteArray

64+ [A] class Unbox a

65+ [A] class Serialize a

66+ [A] SerializeConfig

67+ [A] MutByteArray

68+ [A] unpin :: MutByteArray -> IO MutByteArray

69+ [A] sizeOf :: (Unbox a, SizeOfRep (Rep a)) => Proxy a -> Int

70+ [A] serializeAt :: Serialize a => Int -> MutByteArray -> a -> IO Int

71+ [A] pokeAt :: (Unbox a, Generic a, PokeRep (Rep a)) => Int -> MutByteArray -> a -> IO ()

72+ [A] pinnedNew :: Int -> IO MutByteArray

73+ [A] pin :: MutByteArray -> IO MutByteArray

74+ [A] peekAt :: (Unbox a, Generic a, PeekRep (Rep a)) => Int -> MutByteArray -> IO a

75+ [A] new :: Int -> IO MutByteArray

76+ [A] isPinned :: MutByteArray -> Bool

77+ [A] inlineSerializeAt :: Maybe Inline -> SerializeConfig -> SerializeConfig

78+ [A] inlineDeserializeAt :: Maybe Inline -> SerializeConfig -> SerializeConfig

79+ [A] inlineAddSizeTo :: Maybe Inline -> SerializeConfig -> SerializeConfig

80+ [A] deserializeAt :: Serialize a => Int -> MutByteArray -> Int -> IO (Int, a)

81+ [A] deriveUnbox :: Q [Dec] -> Q [Dec]

82+ [A] deriveSerializeWith :: (SerializeConfig -> SerializeConfig) -> Q [Dec] -> Q [Dec]

83+ [A] deriveSerialize :: Q [Dec] -> Q [Dec]

84+ [A] addSizeTo :: Serialize a => Int -> a -> Int

85+[C] Streamly.Data.MutArray.Generic

86+ [A] write :: MonadIO m => Fold m a (MutArray a)

87+ [A] readRev :: MonadIO m => MutArray a -> Stream m a

88+ [A] read :: MonadIO m => MutArray a -> Stream m a

89+ [A] putIndexUnsafe :: forall m a. MonadIO m => Int -> MutArray a -> a -> m ()

90+ [A] modifyIndexUnsafe :: MonadIO m => Int -> MutArray a -> (a -> (a, b)) -> m b

91+ [A] length :: MutArray a -> Int

92+ [A] getIndexUnsafe :: MonadIO m => Int -> MutArray a -> m a

93+ [C] getIndex

94+ [O] getIndex :: MonadIO m => Int -> MutArray a -> m a

95+ [N] getIndex :: MonadIO m => Int -> MutArray a -> m (Maybe a)

96+ [A] fromListN :: MonadIO m => Int -> [a] -> m (MutArray a)

97+ [A] fromList :: MonadIO m => [a] -> m (MutArray a)

98+[C] Streamly.Data.MutArray

99+ [A] unpin :: MutArray a -> IO (MutArray a)

100+ [A] readRev :: forall m a. (MonadIO m, Unbox a) => MutArray a -> Stream m a

101+ [A] read :: forall m a. (MonadIO m, Unbox a) => MutArray a -> Stream m a

102+ [A] putIndexUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> a -> m ()

103+ [D] pokeByteIndex :: Unbox a => Int -> MutByteArray -> a -> IO ()

104+ [A] pokeAt :: (Unbox a, Generic a, PokeRep (Rep a)) => Int -> MutByteArray -> a -> IO ()

105+ [A] pinnedNew :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a)

106+ [A] pin :: MutArray a -> IO (MutArray a)

107+ [D] peekByteIndex :: Unbox a => Int -> MutByteArray -> IO a

108+ [A] peekAt :: (Unbox a, Generic a, PeekRep (Rep a)) => Int -> MutByteArray -> IO a

109+ [D] newPinned :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a)

110+ [A] modifyIndexUnsafe :: forall m a b. (MonadIO m, Unbox a) => Int -> MutArray a -> (a -> (a, b)) -> m b

111+ [A] modifyIndex :: forall m a b. (MonadIO m, Unbox a) => Int -> MutArray a -> (a -> (a, b)) -> m b

112+ [A] modify :: forall m a. (MonadIO m, Unbox a) => MutArray a -> (a -> a) -> m ()

113+ [A] isPinned :: MutArray a -> Bool

114+ [A] getIndexUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m a

115+ [C] getIndex

116+ [O] getIndex :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m a

117+ [N] getIndex :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (Maybe a)

118+[C] Streamly.Data.Array.Generic

119+ [A] toList :: Array a -> [a]

120+ [A] getIndex :: Int -> Array a -> Maybe a

121+[C] Streamly.Data.Array

122+ [A] unpin :: Array a -> IO (Array a)

123+ [D] pokeByteIndex :: Unbox a => Int -> MutByteArray -> a -> IO ()

124+ [A] pokeAt :: (Unbox a, Generic a, PokeRep (Rep a)) => Int -> MutByteArray -> a -> IO ()

125+ [A] pin :: Array a -> IO (Array a)

126+ [D] peekByteIndex :: Unbox a => Int -> MutByteArray -> IO a

127+ [A] peekAt :: (Unbox a, Generic a, PeekRep (Rep a)) => Int -> MutByteArray -> IO a

128+ [A] isPinned :: Array a -> Bool

129+[C] Streamly.Console.Stdio

130+

131+---------------------------------

132+Internal API diff

133+---------------------------------

134+

135+[C] Streamly.Internal.Unicode.Stream

136+ [C] writeCharUtf8'

137+ [O] writeCharUtf8' :: Monad m => Fold m Word8 Char

138+ [N] writeCharUtf8' :: Monad m => Parser Word8 m Char

139+[C] Streamly.Internal.Unicode.Parser

140+ [A] number :: Monad m => Parser Char m (Integer, Int)

141+ [A] mkDouble :: Integer -> Int -> Double

142+ [A] doubleParser :: Monad m => Parser Char m (Int, Int)

143+ [C] double

144+ [O] double :: Parser Char m Double

145+ [N] double :: Monad m => Parser Char m Double

146+[R] Streamly.Internal.Serialize.ToBytes

147+[R] Streamly.Internal.Serialize.FromBytes

148+[C] Streamly.Internal.FileSystem.File

149+ [A] writeAppendWith :: (MonadIO m, MonadCatch m) => Int -> FilePath -> Stream m Word8 -> m ()

150+ [A] writeAppendChunks :: (MonadIO m, MonadCatch m) => FilePath -> Stream m (Array a) -> m ()

151+ [A] writeAppendArray :: FilePath -> Array a -> IO ()

152+ [A] writeAppend :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8 -> m ()

153+ [R] appendWith :: (MonadIO m, MonadCatch m) => Int -> FilePath -> Stream m Word8 -> m ()

154+ [R] appendChunks :: (MonadIO m, MonadCatch m) => FilePath -> Stream m (Array a) -> m ()

155+ [R] appendArray :: FilePath -> Array a -> IO ()

156+ [R] append :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8 -> m ()

157+[C] Streamly.Internal.FileSystem.Dir

158+ [C] reader

159+ [O] reader :: MonadIO m => Unfold m FilePath FilePath

160+ [N] reader :: (MonadIO m, MonadCatch m) => Unfold m FilePath FilePath

161+ [C] readFiles

162+ [O] readFiles :: MonadIO m => FilePath -> Stream m FilePath

163+ [N] readFiles :: (MonadIO m, MonadCatch m) => FilePath -> Stream m FilePath

164+ [C] readEitherPaths

165+ [O] readEitherPaths :: MonadIO m => FilePath -> Stream m (Either FilePath FilePath)

166+ [N] readEitherPaths :: (MonadIO m, MonadCatch m) => FilePath -> Stream m (Either FilePath FilePath)

167+ [C] readEither

168+ [O] readEither :: MonadIO m => FilePath -> Stream m (Either FilePath FilePath)

169+ [N] readEither :: (MonadIO m, MonadCatch m) => FilePath -> Stream m (Either FilePath FilePath)

170+ [C] readDirs

171+ [O] readDirs :: MonadIO m => FilePath -> Stream m FilePath

172+ [N] readDirs :: (MonadIO m, MonadCatch m) => FilePath -> Stream m FilePath

173+ [C] read

174+ [O] read :: MonadIO m => FilePath -> Stream m FilePath

175+ [N] read :: (MonadIO m, MonadCatch m) => FilePath -> Stream m FilePath

176+ [C] fileReader

177+ [O] fileReader :: MonadIO m => Unfold m FilePath FilePath

178+ [N] fileReader :: (MonadIO m, MonadCatch m) => Unfold m FilePath FilePath

179+ [C] eitherReaderPaths

180+ [O] eitherReaderPaths :: MonadIO m => Unfold m FilePath (Either FilePath FilePath)

181+ [N] eitherReaderPaths :: (MonadIO m, MonadCatch m) => Unfold m FilePath (Either FilePath FilePath)

182+ [C] eitherReader

183+ [O] eitherReader :: MonadIO m => Unfold m FilePath (Either FilePath FilePath)

184+ [N] eitherReader :: (MonadIO m, MonadCatch m) => Unfold m FilePath (Either FilePath FilePath)

185+ [C] dirReader

186+ [O] dirReader :: MonadIO m => Unfold m FilePath FilePath

187+ [N] dirReader :: (MonadIO m, MonadCatch m) => Unfold m FilePath FilePath

188+[R] Streamly.Internal.Data.Unfold.Type

189+[R] Streamly.Internal.Data.Unfold.Enumeration

190+[C] Streamly.Internal.Data.Unfold

191+ [C] Unfold

192+ [A] Unfold :: (s -> m (Step s b)) -> (a -> m s) -> Unfold m a b

193+ [A] takeWhileMWithInput :: Monad m => (a -> b -> m Bool) -> Unfold m a b -> Unfold m a b

194+ [A] manyInterleave :: Monad m => Unfold m a b -> Unfold m c a -> Unfold m c b

195+ [A] enumerateFromStepIntegral :: (Monad m, Integral a) => Unfold m (a, a) a

196+ [A] crossApplySnd :: Unfold m a b -> Unfold m a c -> Unfold m a c

197+ [A] crossApplyFst :: Unfold m a b -> Unfold m a c -> Unfold m a b

198+ [A] concatMap :: Monad m => (b -> Unfold m a c) -> Unfold m a b -> Unfold m a c

199+[R] Streamly.Internal.Data.Unboxed

200+[C] Streamly.Internal.Data.Time.Units

201+ [R] Streamly.Internal.Data.Unboxed.Unbox

202+ [A] Streamly.Internal.Data.Unbox.Unbox

203+ [A] instance Streamly.Internal.Data.Unbox.Unbox Streamly.Internal.Data.Time.Units.NanoSecond64

204+ [A] instance Streamly.Internal.Data.Unbox.Unbox Streamly.Internal.Data.Time.Units.MilliSecond64

205+ [A] instance Streamly.Internal.Data.Unbox.Unbox Streamly.Internal.Data.Time.Units.MicroSecond64

206+[R] Streamly.Internal.Data.Time.Clock.Type

207+[A] Streamly.Internal.Data.StreamK

208+ [A] CrossStreamK

209+ [A] (FixityR,6)

210+ [A] (FixityR,6)

211+ [A] (FixityR,6)

212+ [A] (FixityR,5)

213+ [A] (FixityR,5)

214+ [A] (FixityR,6)

215+ [A] (FixityR,5)

216+ [A] StreamK

217+ [A] MkStream :: (forall r. State StreamK m a -> (a -> StreamK m a -> m r) -> (a -> m r) -> m r -> m r) -> StreamK m a

218+ [A] zipWithM :: Monad m => (a -> b -> m c) -> StreamK m a -> StreamK m b -> StreamK m c

219+ [A] zipWith :: Monad m => (a -> b -> c) -> StreamK m a -> StreamK m b -> StreamK m c

220+ [A] unfoldrMWith :: Monad m => (m a -> StreamK m a -> StreamK m a) -> (b -> m (Maybe (a, b))) -> b -> StreamK m a

221+ [A] unfoldrM :: Monad m => (b -> m (Maybe (a, b))) -> b -> StreamK m a

222+ [A] unfoldr :: (b -> Maybe (a, b)) -> b -> StreamK m a

223+ [A] uncons :: Applicative m => StreamK m a -> m (Maybe (a, StreamK m a))

224+ [A] unShare :: StreamK m a -> StreamK m a

225+ [A] unCross :: CrossStreamK m a -> StreamK m a

226+ [A] toStream :: Applicative m => StreamK m a -> Stream m a

227+ [A] toList :: Monad m => StreamK m a -> m [a]

228+ [A] the :: (Eq a, Monad m) => StreamK m a -> m (Maybe a)

229+ [A] takeWhile :: (a -> Bool) -> StreamK m a -> StreamK m a

230+ [A] take :: Int -> StreamK m a -> StreamK m a

231+ [A] tail :: Applicative m => StreamK m a -> m (Maybe (StreamK m a))

232+ [A] sortBy :: Monad m => (a -> a -> Ordering) -> StreamK m a -> StreamK m a

233+ [A] sequence :: Monad m => StreamK m (m a) -> StreamK m a

234+ [A] scanlx' :: (x -> a -> x) -> x -> (x -> b) -> StreamK m a -> StreamK m b

235+ [A] scanl' :: (b -> a -> b) -> b -> StreamK m a -> StreamK m b

236+ [A] reverse :: StreamK m a -> StreamK m a

237+ [A] replicateMWith :: (m a -> StreamK m a -> StreamK m a) -> Int -> m a -> StreamK m a

238+ [A] replicateM :: Monad m => Int -> m a -> StreamK m a

239+ [A] replicate :: Int -> a -> StreamK m a

240+ [A] repeatMWith :: (m a -> t m a -> t m a) -> m a -> t m a

241+ [A] repeatM :: Monad m => m a -> StreamK m a

242+ [A] repeat :: a -> StreamK m a

243+ [A] parseDBreak :: Monad m => Parser a m b -> StreamK m a -> m (Either ParseError b, StreamK m a)

244+ [A] parseD :: Monad m => Parser a m b -> StreamK m a -> m (Either ParseError b)

245+ [A] parseChunksGeneric :: Monad m => ParserK (Array a) m b -> StreamK m (Array a) -> m (Either ParseError b)

246+ [A] parseChunks :: (Monad m, Unbox a) => ParserK (Array a) m b -> StreamK m (Array a) -> m (Either ParseError b)

247+ [A] parseBreakChunksGeneric :: forall m a b. Monad m => ParserK (Array a) m b -> StreamK m (Array a) -> m (Either ParseError b, StreamK m (Array a))

248+ [A] parseBreakChunks :: (Monad m, Unbox a) => ParserK (Array a) m b -> StreamK m (Array a) -> m (Either ParseError b, StreamK m (Array a))

249+ [A] parseBreak :: forall m a b. Monad m => ParserK a m b -> StreamK m a -> m (Either ParseError b, StreamK m a)

250+ [A] parse :: Monad m => ParserK a m b -> StreamK m a -> m (Either ParseError b)

251+ [A] null :: Monad m => StreamK m a -> m Bool

252+ [A] notElem :: (Monad m, Eq a) => a -> StreamK m a -> m Bool

253+ [A] nilM :: Applicative m => m b -> StreamK m a

254+ [A] nil :: StreamK m a

255+ [A] mkStream :: (forall r. State StreamK m a -> (a -> StreamK m a -> m r) -> (a -> m r) -> m r -> m r) -> StreamK m a

256+ [A] mkCross :: StreamK m a -> CrossStreamK m a

257+ [A] minimumBy :: Monad m => (a -> a -> Ordering) -> StreamK m a -> m (Maybe a)

258+ [A] minimum :: (Monad m, Ord a) => StreamK m a -> m (Maybe a)

259+ [A] mfix :: Monad m => (m a -> StreamK m a) -> StreamK m a

260+ [A] mergeMapWith :: (StreamK m b -> StreamK m b -> StreamK m b) -> (a -> StreamK m b) -> StreamK m a -> StreamK m b

261+ [A] mergeIterateWith :: (StreamK m a -> StreamK m a -> StreamK m a) -> (a -> StreamK m a) -> StreamK m a -> StreamK m a

262+ [A] mergeByM :: Monad m => (a -> a -> m Ordering) -> StreamK m a -> StreamK m a -> StreamK m a

263+ [A] mergeBy :: (a -> a -> Ordering) -> StreamK m a -> StreamK m a -> StreamK m a

264+ [A] maximumBy :: Monad m => (a -> a -> Ordering) -> StreamK m a -> m (Maybe a)

265+ [A] maximum :: (Monad m, Ord a) => StreamK m a -> m (Maybe a)

266+ [A] mapMaybe :: (a -> Maybe b) -> StreamK m a -> StreamK m b

267+ [A] mapM_ :: Monad m => (a -> m b) -> StreamK m a -> m ()

268+ [A] mapMWith :: (m b -> StreamK m b -> StreamK m b) -> (a -> m b) -> StreamK m a -> StreamK m b

269+ [A] mapMSerial :: Monad m => (a -> m b) -> StreamK m a -> StreamK m b

270+ [A] mapM :: Monad m => (a -> m b) -> StreamK m a -> StreamK m b

271+ [A] map :: (a -> b) -> StreamK m a -> StreamK m b

272+ [A] lookup :: (Monad m, Eq a) => a -> StreamK m (a, b) -> m (Maybe b)

273+ [A] liftInner :: (Monad m, MonadTrans t, Monad (t m)) => StreamK m a -> StreamK (t m) a

274+ [A] last :: Monad m => StreamK m a -> m (Maybe a)

275+ [A] iterateMWith :: Monad m => (m a -> StreamK m a -> StreamK m a) -> (a -> m a) -> m a -> StreamK m a

276+ [A] iterateM :: Monad m => (a -> m a) -> m a -> StreamK m a

277+ [A] iterate :: (a -> a) -> a -> StreamK m a

278+ [A] intersperseM :: Monad m => m a -> StreamK m a -> StreamK m a

279+ [A] intersperse :: Monad m => a -> StreamK m a -> StreamK m a

280+ [A] interleaveMin :: StreamK m a -> StreamK m a -> StreamK m a

281+ [A] interleaveFst :: StreamK m a -> StreamK m a -> StreamK m a

282+ [A] interleave :: StreamK m a -> StreamK m a -> StreamK m a

283+ [A] insertBy :: (a -> a -> Ordering) -> a -> StreamK m a -> StreamK m a

284+ [A] init :: Applicative m => StreamK m a -> m (Maybe (StreamK m a))

285+ [A] hoist :: (Monad m, Monad n) => (forall x. m x -> n x) -> StreamK m a -> StreamK n a

286+ [A] head :: Monad m => StreamK m a -> m (Maybe a)

287+ [A] handle :: (MonadCatch m, Exception e) => (e -> m (StreamK m a)) -> StreamK m a -> StreamK m a

288+ [A] fromYieldK :: YieldK m a -> StreamK m a

289+ [A] fromStream :: Monad m => Stream m a -> StreamK m a

290+ [A] fromStopK :: StopK m -> StreamK m a

291+ [A] fromPure :: a -> StreamK m a

292+ [A] fromList :: [a] -> StreamK m a

293+ [A] fromIndicesMWith :: (m a -> StreamK m a -> StreamK m a) -> (Int -> m a) -> StreamK m a

294+ [A] fromIndicesM :: Monad m => (Int -> m a) -> StreamK m a

295+ [A] fromIndices :: (Int -> a) -> StreamK m a

296+ [A] fromFoldableM :: (Foldable f, Monad m) => f (m a) -> StreamK m a

297+ [A] fromFoldable :: Foldable f => f a -> StreamK m a

298+ [A] fromEffect :: Monad m => m a -> StreamK m a

299+ [A] foldrT :: (Monad m, Monad (s m), MonadTrans s) => (a -> s m b -> s m b) -> s m b -> StreamK m a -> s m b

300+ [A] foldrSShared :: (a -> StreamK m b -> StreamK m b) -> StreamK m b -> StreamK m a -> StreamK m b

301+ [A] foldrSM :: Monad m => (m a -> StreamK m b -> StreamK m b) -> StreamK m b -> StreamK m a -> StreamK m b

302+ [A] foldrS :: (a -> StreamK m b -> StreamK m b) -> StreamK m b -> StreamK m a -> StreamK m b

303+ [A] foldrM :: (a -> m b -> m b) -> m b -> StreamK m a -> m b

304+ [A] foldr1 :: Monad m => (a -> a -> a) -> StreamK m a -> m (Maybe a)

305+ [A] foldr :: Monad m => (a -> b -> b) -> b -> StreamK m a -> m b

306+ [A] foldlx' :: forall m a b x. Monad m => (x -> a -> x) -> x -> (x -> b) -> StreamK m a -> m b

307+ [A] foldlT :: (Monad m, Monad (s m), MonadTrans s) => (s m b -> a -> s m b) -> s m b -> StreamK m a -> s m b

308+ [A] foldlS :: (StreamK m b -> a -> StreamK m b) -> StreamK m b -> StreamK m a -> StreamK m b

309+ [A] foldlMx' :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> StreamK m a -> m b

310+ [A] foldlM' :: Monad m => (b -> a -> m b) -> m b -> StreamK m a -> m b

311+ [A] foldl' :: Monad m => (b -> a -> b) -> b -> StreamK m a -> m b

312+ [A] foldStreamShared :: State StreamK m a -> (a -> StreamK m a -> m r) -> (a -> m r) -> m r -> StreamK m a -> m r

313+ [A] foldStream :: State StreamK m a -> (a -> StreamK m a -> m r) -> (a -> m r) -> m r -> StreamK m a -> m r

314+ [A] foldEither :: Monad m => Fold m a b -> StreamK m a -> m (Either (Fold m a b) (b, StreamK m a))

315+ [A] foldConcat :: Monad m => Producer m a b -> Fold m b c -> StreamK m a -> m (c, StreamK m a)

316+ [A] foldBreak :: Monad m => Fold m a b -> StreamK m a -> m (b, StreamK m a)

317+ [A] fold :: Monad m => Fold m a b -> StreamK m a -> m b

318+ [A] findM :: Monad m => (a -> m Bool) -> StreamK m a -> m (Maybe a)

319+ [A] findIndices :: (a -> Bool) -> StreamK m a -> StreamK m Int

320+ [A] find :: Monad m => (a -> Bool) -> StreamK m a -> m (Maybe a)

321+ [A] filter :: (a -> Bool) -> StreamK m a -> StreamK m a

322+ [A] evalStateT :: Monad m => m s -> StreamK (StateT s m) a -> StreamK m a

323+ [A] elem :: (Monad m, Eq a) => a -> StreamK m a -> m Bool

324+ [A] dropWhile :: (a -> Bool) -> StreamK m a -> StreamK m a

325+ [A] drop :: Int -> StreamK m a -> StreamK m a

326+ [A] drain :: Monad m => StreamK m a -> m ()

327+ [A] deleteBy :: (a -> a -> Bool) -> a -> StreamK m a -> StreamK m a

328+ [A] crossWith :: Monad m => (a -> b -> c) -> StreamK m a -> StreamK m b -> StreamK m c

329+ [A] crossApplyWith :: (StreamK m b -> StreamK m b -> StreamK m b) -> StreamK m (a -> b) -> StreamK m a -> StreamK m b

330+ [A] crossApplySnd :: StreamK m a -> StreamK m b -> StreamK m b

331+ [A] crossApplyFst :: StreamK m a -> StreamK m b -> StreamK m a

332+ [A] crossApply :: StreamK m (a -> b) -> StreamK m a -> StreamK m b

333+ [A] cross :: Monad m => StreamK m a -> StreamK m b -> StreamK m (a, b)

334+ [A] consMBy :: Monad m => (StreamK m a -> StreamK m a -> StreamK m a) -> m a -> StreamK m a -> StreamK m a

335+ [A] consM :: Monad m => m a -> StreamK m a -> StreamK m a

336+ [A] consK :: YieldK m a -> StreamK m a -> StreamK m a

337+ [A] cons :: a -> StreamK m a -> StreamK m a

338+ [A] conjoin :: Monad m => StreamK m a -> StreamK m a -> StreamK m a

339+ [A] concatMapWith :: (StreamK m b -> StreamK m b -> StreamK m b) -> (a -> StreamK m b) -> StreamK m a -> StreamK m b

340+ [A] concatMapEffect :: Monad m => (b -> StreamK m a) -> m b -> StreamK m a

341+ [A] concatMap :: (a -> StreamK m b) -> StreamK m a -> StreamK m b

342+ [A] concatIterateWith :: (StreamK m a -> StreamK m a -> StreamK m a) -> (a -> StreamK m a) -> StreamK m a -> StreamK m a

343+ [A] concatIterateScanWith :: Monad m => (StreamK m a -> StreamK m a -> StreamK m a) -> (b -> a -> m (b, StreamK m a)) -> m b -> StreamK m a -> StreamK m a

344+ [A] concatIterateLeftsWith :: b ~ Either a c => (StreamK m b -> StreamK m b -> StreamK m b) -> (a -> StreamK m b) -> StreamK m b -> StreamK m b

345+ [A] concatEffect :: Monad m => m (StreamK m a) -> StreamK m a

346+ [A] buildSM :: Monad m => ((m a -> StreamK m a -> StreamK m a) -> StreamK m a -> StreamK m a) -> StreamK m a

347+ [A] buildS :: ((a -> StreamK m a -> StreamK m a) -> StreamK m a -> StreamK m a) -> StreamK m a

348+ [A] buildM :: Monad m => (forall r. (a -> StreamK m a -> m r) -> (a -> m r) -> m r -> m r) -> StreamK m a

349+ [A] build :: forall m a. (forall b. (a -> b -> b) -> b -> b) -> StreamK m a

350+ [A] bracketIO :: (MonadIO m, MonadCatch m) => IO b -> (b -> IO c) -> (b -> StreamK m a) -> StreamK m a

351+ [A] bindWith :: (StreamK m b -> StreamK m b -> StreamK m b) -> StreamK m a -> (a -> StreamK m b) -> StreamK m b

352+ [A] before :: Monad m => m b -> StreamK m a -> StreamK m a

353+ [A] augmentSM :: Monad m => ((m a -> StreamK m a -> StreamK m a) -> StreamK m a -> StreamK m a) -> StreamK m a -> StreamK m a

354+ [A] augmentS :: ((a -> StreamK m a -> StreamK m a) -> StreamK m a -> StreamK m a) -> StreamK m a -> StreamK m a

355+ [A] append :: StreamK m a -> StreamK m a -> StreamK m a

356+ [A] any :: Monad m => (a -> Bool) -> StreamK m a -> m Bool

357+ [A] all :: Monad m => (a -> Bool) -> StreamK m a -> m Bool

358+ [A] (.:) :: a -> StreamK m a -> StreamK m a

359+ [A] (!!) :: Monad m => StreamK m a -> Int -> m (Maybe a)

360+[R] Streamly.Internal.Data.Stream.StreamK.Type

361+[R] Streamly.Internal.Data.Stream.StreamK.Transformer

362+[R] Streamly.Internal.Data.Stream.StreamK

363+[R] Streamly.Internal.Data.Stream.StreamD.Type

364+[R] Streamly.Internal.Data.Stream.StreamD.Transformer

365+[R] Streamly.Internal.Data.Stream.StreamD.Transform

366+[R] Streamly.Internal.Data.Stream.StreamD.Top

367+[R] Streamly.Internal.Data.Stream.StreamD.Step

368+[R] Streamly.Internal.Data.Stream.StreamD.Nesting

369+[R] Streamly.Internal.Data.Stream.StreamD.Lift

370+[R] Streamly.Internal.Data.Stream.StreamD.Generate

371+[R] Streamly.Internal.Data.Stream.StreamD.Exception

372+[R] Streamly.Internal.Data.Stream.StreamD.Eliminate

373+[R] Streamly.Internal.Data.Stream.StreamD.Container

374+[D] Streamly.Internal.Data.Stream.StreamD

375+[R] Streamly.Internal.Data.Stream.Common

376+[R] Streamly.Internal.Data.Stream.Chunked

377+[C] Streamly.Internal.Data.Stream

378+ [A] class Enum a => Enumerable a

379+ [A] Stream

380+ [A] UnStream :: (State StreamK m a -> s -> m (Step s a)) -> s -> Stream m a

381+ [A] Step

382+ [A] Yield :: a -> s -> Step s a

383+ [A] Stop :: Step s a

384+ [A] Skip :: s -> Step s a

385+ [A] InterleaveState

386+ [A] InterleaveSecondOnly :: s2 -> InterleaveState s1 s2

387+ [A] InterleaveSecond :: s1 -> s2 -> InterleaveState s1 s2

388+ [A] InterleaveFirstOnly :: s1 -> InterleaveState s1 s2

389+ [A] InterleaveFirst :: s1 -> s2 -> InterleaveState s1 s2

390+ [A] FoldManyPost

391+ [A] FoldManyPostYield :: b -> FoldManyPost s fs b a -> FoldManyPost s fs b a

392+ [A] FoldManyPostStart :: s -> FoldManyPost s fs b a

393+ [A] FoldManyPostLoop :: s -> fs -> FoldManyPost s fs b a

394+ [A] FoldManyPostDone :: FoldManyPost s fs b a

395+ [A] FoldMany

396+ [A] FoldManyYield :: b -> FoldMany s fs b a -> FoldMany s fs b a

397+ [A] FoldManyStart :: s -> FoldMany s fs b a

398+ [A] FoldManyLoop :: s -> fs -> FoldMany s fs b a

399+ [A] FoldManyFirst :: fs -> s -> FoldMany s fs b a

400+ [A] FoldManyDone :: FoldMany s fs b a

401+ [A] CrossStream

402+ [A] ConcatUnfoldInterleaveState

403+ [A] ConcatUnfoldInterleaveOuter :: o -> [i] -> ConcatUnfoldInterleaveState o i

404+ [A] ConcatUnfoldInterleaveInnerR :: [i] -> [i] -> ConcatUnfoldInterleaveState o i

405+ [A] ConcatUnfoldInterleaveInnerL :: [i] -> [i] -> ConcatUnfoldInterleaveState o i

406+ [A] ConcatUnfoldInterleaveInner :: o -> [i] -> ConcatUnfoldInterleaveState o i

407+ [A] ConcatMapUState

408+ [A] ConcatMapUOuter :: o -> ConcatMapUState o i

409+ [A] ConcatMapUInner :: o -> i -> ConcatMapUState o i

410+ [A] AppendState

411+ [A] AppendSecond :: s2 -> AppendState s1 s2

412+ [A] AppendFirst :: s1 -> AppendState s1 s2

413+ [A] pattern Stream :: (State StreamK m a -> s -> m (Step s a)) -> s -> Stream m a

414+ [A] zipWithM :: Monad m => (a -> b -> m c) -> Stream m a -> Stream m b -> Stream m c

415+ [A] zipWith :: Monad m => (a -> b -> c) -> Stream m a -> Stream m b -> Stream m c

416+ [A] wordsBy :: Monad m => (a -> Bool) -> Fold m a b -> Stream m a -> Stream m b

417+ [A] with :: Monad m => (Stream m a -> Stream m (s, a)) -> (((s, a) -> b) -> Stream m (s, a) -> Stream m (s, a)) -> ((s, a) -> b) -> Stream m a -> Stream m a

418+ [A] usingStateT :: Monad m => m s -> (Stream (StateT s m) a -> Stream (StateT s m) a) -> Stream m a -> Stream m a

419+ [A] usingReaderT :: Monad m => m r -> (Stream (ReaderT r m) a -> Stream (ReaderT r m) a) -> Stream m a -> Stream m a

420+ [A] uniqBy :: Monad m => (a -> a -> Bool) -> Stream m a -> Stream m a

421+ [A] uniq :: (Eq a, Monad m) => Stream m a -> Stream m a

422+ [A] unionWithStreamGenericBy :: MonadIO m => (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a

423+ [A] unionWithStreamAscBy :: (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a

424+ [A] unfoldrM :: Monad m => (s -> m (Maybe (a, s))) -> s -> Stream m a

425+ [A] unfoldr :: Monad m => (s -> Maybe (a, s)) -> s -> Stream m a

426+ [A] unfoldRoundRobin :: Monad m => Unfold m a b -> Stream m a -> Stream m b

427+ [A] unfoldMany :: Monad m => Unfold m a b -> Stream m a -> Stream m b

428+ [A] unfoldIterateDfs :: Monad m => Unfold m a a -> Stream m a -> Stream m a

429+ [A] unfoldIterateBfsRev :: Monad m => Unfold m a a -> Stream m a -> Stream m a

430+ [A] unfoldIterateBfs :: Monad m => Unfold m a a -> Stream m a -> Stream m a

431+ [A] unfoldInterleave :: Monad m => Unfold m a b -> Stream m a -> Stream m b

432+ [A] unfold :: Applicative m => Unfold m a b -> a -> Stream m b

433+ [A] uncons :: Monad m => Stream m a -> m (Maybe (a, Stream m a))

434+ [A] unCross :: CrossStream m a -> Stream m a

435+ [A] transform :: Monad m => Pipe m a b -> Stream m a -> Stream m b

436+ [A] trace_ :: Monad m => m b -> Stream m a -> Stream m a

437+ [A] trace :: Monad m => (a -> m b) -> Stream m a -> Stream m a

438+ [A] toStreamK :: Monad m => Stream m a -> StreamK m a

439+ [A] toListRev :: Monad m => Stream m a -> m [a]

440+ [A] toList :: Monad m => Stream m a -> m [a]

441+ [A] timestamped :: MonadIO m => Stream m a -> Stream m (AbsTime, a)

442+ [A] timestampWith :: MonadIO m => Double -> Stream m a -> Stream m (AbsTime, a)

443+ [A] timesWith :: MonadIO m => Double -> Stream m (AbsTime, RelTime64)

444+ [A] times :: MonadIO m => Stream m (AbsTime, RelTime64)

445+ [A] timeout :: AbsTime -> t m ()

446+ [A] timeIndexed :: MonadIO m => Stream m a -> Stream m (RelTime64, a)

447+ [A] timeIndexWith :: MonadIO m => Double -> Stream m a -> Stream m (RelTime64, a)

448+ [A] the :: (Eq a, Monad m) => Stream m a -> m (Maybe a)

449+ [A] tapOffsetEvery :: Monad m => Int -> Int -> Fold m a b -> Stream m a -> Stream m a

450+ [A] tap :: Monad m => Fold m a b -> Stream m a -> Stream m a

451+ [A] takeWhileM :: Monad m => (a -> m Bool) -> Stream m a -> Stream m a

452+ [A] takeWhileLast :: (a -> Bool) -> Stream m a -> Stream m a

453+ [A] takeWhileAround :: (a -> Bool) -> Stream m a -> Stream m a

454+ [A] takeWhile :: Monad m => (a -> Bool) -> Stream m a -> Stream m a

455+ [A] takeEndByM :: Monad m => (a -> m Bool) -> Stream m a -> Stream m a

456+ [A] takeEndBy :: Monad m => (a -> Bool) -> Stream m a -> Stream m a

457+ [A] take :: Applicative m => Int -> Stream m a -> Stream m a

458+ [A] tail :: Monad m => Stream m a -> m (Maybe (Stream m a))

459+ [A] stripSuffixUnbox :: (MonadIO m, Eq a, Unbox a) => Stream m a -> Stream m a -> m (Maybe (Stream m a))

460+ [A] stripSuffix :: (Monad m, Eq a) => Stream m a -> Stream m a -> m (Maybe (Stream m a))

461+ [A] stripPrefix :: (Monad m, Eq a) => Stream m a -> Stream m a -> m (Maybe (Stream m a))

462+ [A] strideFromThen :: Monad m => Int -> Int -> Stream m a -> Stream m a

463+ [A] splitOnSuffixSeqAny :: [Array a] -> Fold m a b -> Stream m a -> Stream m b

464+ [A] splitOnSuffixSeq :: forall m a b. (MonadIO m, Storable a, Unbox a, Enum a, Eq a) => Bool -> Array a -> Fold m a b -> Stream m a -> Stream m b

465+ [A] splitOnSeq :: forall m a b. (MonadIO m, Storable a, Unbox a, Enum a, Eq a) => Array a -> Fold m a b -> Stream m a -> Stream m b

466+ [A] splitOnPrefix :: (a -> Bool) -> Fold m a b -> Stream m a -> Stream m b

467+ [A] splitOnAny :: [Array a] -> Fold m a b -> Stream m a -> Stream m b

468+ [A] splitOn :: Monad m => (a -> Bool) -> Fold m a b -> Stream m a -> Stream m b

469+ [A] splitInnerBySuffix :: (Monad m, Eq (f a), Monoid (f a)) => (f a -> m (f a, Maybe (f a))) -> (f a -> f a -> m (f a)) -> Stream m (f a) -> Stream m (f a)

470+ [A] splitInnerBy :: Monad m => (f a -> m (f a, Maybe (f a))) -> (f a -> f a -> m (f a)) -> Stream m (f a) -> Stream m (f a)

471+ [A] slicesBy :: Monad m => (a -> Bool) -> Stream m a -> Stream m (Int, Int)

472+ [A] sliceOnSuffix :: Monad m => (a -> Bool) -> Stream m a -> Stream m (Int, Int)

473+ [A] sequence :: Monad m => Stream m (m a) -> Stream m a

474+ [A] scanlx' :: Monad m => (x -> a -> x) -> x -> (x -> b) -> Stream m a -> Stream m b

475+ [A] scanlMx' :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> Stream m a -> Stream m b

476+ [A] scanlMAfter' :: Monad m => (b -> a -> m b) -> m b -> (b -> m b) -> Stream m a -> Stream m b

477+ [A] scanlM' :: Monad m => (b -> a -> m b) -> m b -> Stream m a -> Stream m b

478+ [A] scanlM :: Monad m => (b -> a -> m b) -> m b -> Stream m a -> Stream m b

479+ [A] scanl1M' :: Monad m => (a -> a -> m a) -> Stream m a -> Stream m a

480+ [A] scanl1M :: Monad m => (a -> a -> m a) -> Stream m a -> Stream m a

481+ [A] scanl1' :: Monad m => (a -> a -> a) -> Stream m a -> Stream m a

482+ [A] scanl1 :: Monad m => (a -> a -> a) -> Stream m a -> Stream m a

483+ [A] scanl' :: Monad m => (b -> a -> b) -> b -> Stream m a -> Stream m b

484+ [A] scanl :: Monad m => (b -> a -> b) -> b -> Stream m a -> Stream m b

485+ [A] scanMaybe :: Monad m => Fold m a (Maybe b) -> Stream m a -> Stream m b

486+ [A] scanMany :: Monad m => Fold m a b -> Stream m a -> Stream m b

487+ [A] scan :: Monad m => Fold m a b -> Stream m a -> Stream m b

488+ [A] runStateT :: Monad m => m s -> Stream (StateT s m) a -> Stream m (s, a)

489+ [A] runReaderT :: Monad m => m s -> Stream (ReaderT s m) a -> Stream m a

490+ [A] runInnerWithState :: Monad m => (forall b. s -> t m b -> m (b, s)) -> m s -> Stream (t m) a -> Stream m (s, a)

491+ [A] runInnerWith :: Monad m => (forall b. t m b -> m b) -> Stream (t m) a -> Stream m a

492+ [A] roundRobin :: Monad m => Stream m a -> Stream m a -> Stream m a

493+ [A] rollingMapM :: Monad m => (Maybe a -> a -> m b) -> Stream m a -> Stream m b

494+ [A] rollingMap2 :: Monad m => (a -> a -> b) -> Stream m a -> Stream m b

495+ [A] rollingMap :: Monad m => (Maybe a -> a -> b) -> Stream m a -> Stream m b

496+ [A] reverseUnbox :: (MonadIO m, Unbox a) => Stream m a -> Stream m a

497+ [A] reverse :: Monad m => Stream m a -> Stream m a

498+ [A] replicateM :: Monad m => Int -> m a -> Stream m a

499+ [A] replicate :: Monad m => Int -> a -> Stream m a

500+ [A] repeated :: Stream m a -> Stream m a

501+ [A] repeatM :: Monad m => m a -> Stream m a

502+ [A] repeat :: Monad m => a -> Stream m a

503+ [A] relTimesWith :: MonadIO m => Double -> Stream m RelTime64

504+ [A] relTimes :: MonadIO m => Stream m RelTime64

505+ [A] refoldMany :: Monad m => Refold m x a b -> m x -> Stream m a -> Stream m b

506+ [A] refoldIterateM :: Monad m => Refold m b a b -> m b -> Stream m a -> Stream m b

507+ [A] reduceIterateBfs :: Monad m => (a -> a -> m a) -> Stream m a -> m (Maybe a)

508+ [A] reassembleBy :: Fold m a b -> (a -> a -> Int) -> Stream m a -> Stream m b

509+ [A] prune :: (a -> Bool) -> Stream m a -> Stream m a

510+ [A] prescanlM' :: Monad m => (b -> a -> m b) -> m b -> Stream m a -> Stream m b

511+ [A] prescanl' :: Monad m => (b -> a -> b) -> b -> Stream m a -> Stream m b

512+ [A] postscanlx' :: Monad m => (x -> a -> x) -> x -> (x -> b) -> Stream m a -> Stream m b

513+ [A] postscanlMx' :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> Stream m a -> Stream m b

514+ [A] postscanlMAfter' :: Monad m => (b -> a -> m b) -> m b -> (b -> m b) -> Stream m a -> Stream m b

515+ [A] postscanlM' :: Monad m => (b -> a -> m b) -> m b -> Stream m a -> Stream m b

516+ [A] postscanlM :: Monad m => (b -> a -> m b) -> m b -> Stream m a -> Stream m b

517+ [A] postscanl' :: Monad m => (a -> b -> a) -> a -> Stream m b -> Stream m a

518+ [A] postscanl :: Monad m => (a -> b -> a) -> a -> Stream m b -> Stream m a

519+ [A] postscan :: Monad m => Fold m a b -> Stream m a -> Stream m b

520+ [A] parseSequence :: Stream m (Parser a m b) -> Stream m a -> Stream m b

521+ [A] parseManyTill :: Parser a m b -> Parser a m x -> Stream m a -> Stream m b

522+ [A] parseManyD :: Monad m => Parser a m b -> Stream m a -> Stream m (Either ParseError b)

523+ [A] parseMany :: Monad m => Parser a m b -> Stream m a -> Stream m (Either ParseError b)

524+ [A] parseIterateD :: Monad m => (b -> Parser a m b) -> b -> Stream m a -> Stream m (Either ParseError b)

525+ [A] parseIterate :: Monad m => (b -> Parser a m b) -> b -> Stream m a -> Stream m (Either ParseError b)

526+ [A] parseD :: Monad m => Parser a m b -> Stream m a -> m (Either ParseError b)

527+ [A] parseBreakD :: Monad m => Parser a m b -> Stream m a -> m (Either ParseError b, Stream m a)

528+ [A] parseBreak :: Monad m => Parser a m b -> Stream m a -> m (Either ParseError b, Stream m a)

529+ [A] parse :: Monad m => Parser a m b -> Stream m a -> m (Either ParseError b)

530+ [A] onException :: MonadCatch m => m b -> Stream m a -> Stream m a

531+ [A] null :: Monad m => Stream m a -> m Bool

532+ [A] nub :: (Monad m, Ord a) => Stream m a -> Stream m a

533+ [A] notElem :: (Monad m, Eq a) => a -> Stream m a -> m Bool

534+ [A] nilM :: Applicative m => m b -> Stream m a

535+ [A] nil :: Applicative m => Stream m a

536+ [A] morphInner :: Monad n => (forall x. m x -> n x) -> Stream m a -> Stream n a

537+ [A] mkCross :: Stream m a -> CrossStream m a

538+ [A] minimumBy :: Monad m => (a -> a -> Ordering) -> Stream m a -> m (Maybe a)

539+ [A] minimum :: (Monad m, Ord a) => Stream m a -> m (Maybe a)

540+ [A] mergeMinBy :: (a -> a -> m Ordering) -> Stream m a -> Stream m a -> Stream m a

541+ [A] mergeFstBy :: (a -> a -> m Ordering) -> Stream m a -> Stream m a -> Stream m a

542+ [A] mergeByM :: Monad m => (a -> a -> m Ordering) -> Stream m a -> Stream m a -> Stream m a

543+ [A] mergeBy :: Monad m => (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a

544+ [A] maximumBy :: Monad m => (a -> a -> Ordering) -> Stream m a -> m (Maybe a)

545+ [A] maximum :: (Monad m, Ord a) => Stream m a -> m (Maybe a)

546+ [A] mapMaybeM :: Monad m => (a -> m (Maybe b)) -> Stream m a -> Stream m b

547+ [A] mapMaybe :: Monad m => (a -> Maybe b) -> Stream m a -> Stream m b

548+ [A] mapM_ :: Monad m => (a -> m b) -> Stream m a -> m ()

549+ [A] mapM :: Monad m => (a -> m b) -> Stream m a -> Stream m b

550+ [A] map :: Monad m => (a -> b) -> Stream m a -> Stream m b

551+ [A] lookup :: (Monad m, Eq a) => a -> Stream m (a, b) -> m (Maybe b)

552+ [A] liftInnerWith :: Monad (t m) => (forall b. m b -> t m b) -> Stream m a -> Stream (t m) a

553+ [A] liftInner :: (Monad m, MonadTrans t, Monad (t m)) => Stream m a -> Stream (t m) a

554+ [A] last :: Monad m => Stream m a -> m (Maybe a)

555+ [A] joinOuterGeneric :: MonadIO m => (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (Maybe a, Maybe b)

556+ [A] joinOuterAscBy :: (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (Maybe a, Maybe b)

557+ [A] joinOuter :: (Ord k, MonadIO m) => Stream m (k, a) -> Stream m (k, b) -> Stream m (k, Maybe a, Maybe b)

558+ [A] joinLeftGeneric :: Monad m => (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)

559+ [A] joinLeftAscBy :: (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)

560+ [A] joinLeft :: (Ord k, Monad m) => Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, Maybe b)

561+ [A] joinInnerGeneric :: Monad m => (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, b)

562+ [A] joinInnerAscBy :: (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, b)

563+ [A] joinInner :: (Monad m, Ord k) => Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, b)

564+ [A] iterateM :: Monad m => (a -> m a) -> m a -> Stream m a

565+ [A] iterate :: Monad m => (a -> a) -> a -> Stream m a

566+ [A] isSuffixOfUnbox :: (MonadIO m, Eq a, Unbox a) => Stream m a -> Stream m a -> m Bool

567+ [A] isSuffixOf :: (Monad m, Eq a) => Stream m a -> Stream m a -> m Bool

568+ [A] isSubsequenceOf :: (Monad m, Eq a) => Stream m a -> Stream m a -> m Bool

569+ [A] isPrefixOf :: (Monad m, Eq a) => Stream m a -> Stream m a -> m Bool

570+ [A] isInfixOf :: (MonadIO m, Eq a, Enum a, Storable a, Unbox a) => Stream m a -> Stream m a -> m Bool

571+ [A] intersperseM_ :: Monad m => m b -> Stream m a -> Stream m a

572+ [A] intersperseMWith :: Int -> m a -> Stream m a -> Stream m a

573+ [A] intersperseMSuffix_ :: Monad m => m b -> Stream m a -> Stream m a

574+ [A] intersperseMSuffixWith :: forall m a. Monad m => Int -> m a -> Stream m a -> Stream m a

575+ [A] intersperseMSuffix :: forall m a. Monad m => m a -> Stream m a -> Stream m a

576+ [A] intersperseMPrefix_ :: Monad m => m b -> Stream m a -> Stream m a

577+ [A] intersperseM :: Monad m => m a -> Stream m a -> Stream m a

578+ [A] intersperse :: Monad m => a -> Stream m a -> Stream m a

579+ [A] intersectBySorted :: Monad m => (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a

580+ [A] interposeSuffixM :: Monad m => m c -> Unfold m b c -> Stream m b -> Stream m c

581+ [A] interposeSuffix :: Monad m => c -> Unfold m b c -> Stream m b -> Stream m c

582+ [A] interposeM :: Monad m => m c -> Unfold m b c -> Stream m b -> Stream m c

583+ [A] interpose :: Monad m => c -> Unfold m b c -> Stream m b -> Stream m c

584+ [A] interleaveMin :: Monad m => Stream m a -> Stream m a -> Stream m a

585+ [A] interleaveFstSuffix :: Monad m => Stream m a -> Stream m a -> Stream m a

586+ [A] interleaveFst :: Monad m => Stream m a -> Stream m a -> Stream m a

587+ [A] interleave :: Monad m => Stream m a -> Stream m a -> Stream m a

588+ [A] intercalateSuffix :: Monad m => Unfold m b c -> b -> Stream m b -> Stream m c

589+ [A] intercalate :: Monad m => Unfold m b c -> b -> Stream m b -> Stream m c

590+ [A] insertBy :: Monad m => (a -> a -> Ordering) -> a -> Stream m a -> Stream m a

591+ [A] indexedR :: Monad m => Int -> Stream m a -> Stream m (Int, a)

592+ [A] indexed :: Monad m => Stream m a -> Stream m (Int, a)

593+ [A] headElse :: Monad m => a -> Stream m a -> m a

594+ [A] head :: Monad m => Stream m a -> m (Maybe a)

595+ [A] handle :: (MonadCatch m, Exception e) => (e -> m (Stream m a)) -> Stream m a -> Stream m a

596+ [A] groupsWhile :: Monad m => (a -> a -> Bool) -> Fold m a b -> Stream m a -> Stream m b

597+ [A] groupsRollingBy :: Monad m => (a -> a -> Bool) -> Fold m a b -> Stream m a -> Stream m b

598+ [A] groupsOf :: Monad m => Int -> Fold m a b -> Stream m a -> Stream m b

599+ [A] gintercalateSuffix :: Monad m => Unfold m a c -> Stream m a -> Unfold m b c -> Stream m b -> Stream m c

600+ [A] gintercalate :: Monad m => Unfold m a c -> Stream m a -> Unfold m b c -> Stream m b -> Stream m c

601+ [A] ghandle :: (MonadCatch m, Exception e) => (e -> Stream m a -> m (Stream m a)) -> Stream m a -> Stream m a

602+ [A] generateM :: Monad m => Int -> (Int -> m a) -> Stream m a

603+ [A] generate :: Monad m => Int -> (Int -> a) -> Stream m a

604+ [A] generalizeInner :: Monad m => Stream Identity a -> Stream m a

605+ [A] gbracket_ :: Monad m => m c -> (c -> m d) -> (c -> e -> Stream m b -> m (Stream m b)) -> (forall s. m s -> m (Either e s)) -> (c -> Stream m b) -> Stream m b

606+ [A] gbracket :: MonadIO m => IO c -> (c -> IO d1) -> (c -> e -> Stream m b -> IO (Stream m b)) -> (c -> IO d2) -> (forall s. m s -> m (Either e s)) -> (c -> Stream m b) -> Stream m b

607+ [A] fromStreamK :: Applicative m => StreamK m a -> Stream m a

608+ [A] fromPure :: Applicative m => a -> Stream m a

609+ [A] fromPtrN :: (Monad m, Storable a) => Int -> Ptr a -> Stream m a

610+ [A] fromPtr :: forall m a. (Monad m, Storable a) => Ptr a -> Stream m a

611+ [A] fromListM :: Monad m => [m a] -> Stream m a

612+ [A] fromList :: Applicative m => [a] -> Stream m a

613+ [A] fromIndicesM :: Monad m => (Int -> m a) -> Stream m a

614+ [A] fromIndices :: Monad m => (Int -> a) -> Stream m a

615+ [A] fromFoldableM :: (Monad m, Foldable f) => f (m a) -> Stream m a

616+ [A] fromFoldable :: (Monad m, Foldable f) => f a -> Stream m a

617+ [A] fromEffect :: Applicative m => m a -> Stream m a

618+ [A] fromByteStr# :: Monad m => Addr# -> Stream m Word8

619+ [A] foldrT :: (Monad m, Monad (t m), MonadTrans t) => (a -> t m b -> t m b) -> t m b -> Stream m a -> t m b

620+ [A] foldrS :: Monad m => (a -> Stream m b -> Stream m b) -> Stream m b -> Stream m a -> Stream m b

621+ [A] foldrMx :: Monad m => (a -> m x -> m x) -> m x -> (m x -> m b) -> Stream m a -> m b

622+ [A] foldrM :: Monad m => (a -> m b -> m b) -> m b -> Stream m a -> m b

623+ [A] foldr1 :: Monad m => (a -> a -> a) -> Stream m a -> m (Maybe a)

624+ [A] foldr :: Monad m => (a -> b -> b) -> b -> Stream m a -> m b

625+ [A] foldlx' :: Monad m => (x -> a -> x) -> x -> (x -> b) -> Stream m a -> m b

626+ [A] foldlT :: (Monad m, Monad (s m), MonadTrans s) => (s m b -> a -> s m b) -> s m b -> Stream m a -> s m b

627+ [A] foldlS :: Monad m => (Stream m b -> a -> Stream m b) -> Stream m b -> Stream m a -> Stream m b

628+ [A] foldlMx' :: Monad m => (x -> a -> m x) -> m x -> (x -> m b) -> Stream m a -> m b

629+ [A] foldlM' :: Monad m => (b -> a -> m b) -> m b -> Stream m a -> m b

630+ [A] foldl' :: Monad m => (b -> a -> b) -> b -> Stream m a -> m b

631+ [A] foldSequence :: Stream m (Fold m a b) -> Stream m a -> Stream m b

632+ [A] foldManyPost :: Monad m => Fold m a b -> Stream m a -> Stream m b

633+ [A] foldMany :: Monad m => Fold m a b -> Stream m a -> Stream m b

634+ [A] foldIterateM :: Monad m => (b -> m (Fold m a b)) -> m b -> Stream m a -> Stream m b

635+ [A] foldIterateBfs :: Fold m a (Either a a) -> Stream m a -> m (Maybe a)

636+ [A] foldEither :: Monad m => Fold m a b -> Stream m a -> m (Either (Fold m a b) (b, Stream m a))

637+ [A] foldBreak :: Monad m => Fold m a b -> Stream m a -> m (b, Stream m a)

638+ [A] foldAddLazy :: Monad m => Fold m a b -> Stream m a -> Fold m a b

639+ [A] foldAdd :: Monad m => Fold m a b -> Stream m a -> m (Fold m a b)

640+ [A] fold :: Monad m => Fold m a b -> Stream m a -> m b

641+ [A] findM :: Monad m => (a -> m Bool) -> Stream m a -> m (Maybe a)

642+ [A] findIndices :: Monad m => (a -> Bool) -> Stream m a -> Stream m Int

643+ [A] find :: Monad m => (a -> Bool) -> Stream m a -> m (Maybe a)

644+ [A] finallyUnsafe :: MonadCatch m => m b -> Stream m a -> Stream m a

645+ [A] finallyIO :: (MonadIO m, MonadCatch m) => IO b -> Stream m a -> Stream m a

646+ [A] filterM :: Monad m => (a -> m Bool) -> Stream m a -> Stream m a

647+ [A] filterInStreamGenericBy :: Monad m => (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a

648+ [A] filterInStreamAscBy :: Monad m => (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a

649+ [A] filter :: Monad m => (a -> Bool) -> Stream m a -> Stream m a

650+ [A] evalStateT :: Monad m => m s -> Stream (StateT s m) a -> Stream m a

651+ [A] eqBy :: Monad m => (a -> b -> Bool) -> Stream m a -> Stream m b -> m Bool

652+ [A] enumerateTo :: (Monad m, Bounded a, Enumerable a) => a -> Stream m a

653+ [A] enumerateFromToSmall :: (Monad m, Enum a) => a -> a -> Stream m a

654+ [A] enumerateFromToIntegral :: (Monad m, Integral a) => a -> a -> Stream m a

655+ [A] enumerateFromToFractional :: (Monad m, Fractional a, Ord a) => a -> a -> Stream m a

656+ [A] enumerateFromTo :: (Enumerable a, Monad m) => a -> a -> Stream m a

657+ [A] enumerateFromThenToSmall :: (Monad m, Enum a) => a -> a -> a -> Stream m a

658+ [A] enumerateFromThenToIntegral :: (Monad m, Integral a) => a -> a -> a -> Stream m a

659+ [A] enumerateFromThenToFractional :: (Monad m, Fractional a, Ord a) => a -> a -> a -> Stream m a

660+ [A] enumerateFromThenTo :: (Enumerable a, Monad m) => a -> a -> a -> Stream m a

661+ [A] enumerateFromThenSmallBounded :: (Monad m, Enumerable a, Bounded a) => a -> a -> Stream m a

662+ [A] enumerateFromThenNum :: (Monad m, Num a) => a -> a -> Stream m a

663+ [A] enumerateFromThenIntegral :: (Monad m, Integral a, Bounded a) => a -> a -> Stream m a

664+ [A] enumerateFromThenFractional :: (Monad m, Fractional a) => a -> a -> Stream m a

665+ [A] enumerateFromThen :: (Enumerable a, Monad m) => a -> a -> Stream m a

666+ [A] enumerateFromStepNum :: (Monad m, Num a) => a -> a -> Stream m a

667+ [A] enumerateFromStepIntegral :: (Integral a, Monad m) => a -> a -> Stream m a

668+ [A] enumerateFromNum :: (Monad m, Num a) => a -> Stream m a

669+ [A] enumerateFromIntegral :: (Monad m, Integral a, Bounded a) => a -> Stream m a

670+ [A] enumerateFromFractional :: (Monad m, Fractional a) => a -> Stream m a

671+ [A] enumerateFromBounded :: (Monad m, Enumerable a, Bounded a) => a -> Stream m a

672+ [A] enumerateFrom :: (Enumerable a, Monad m) => a -> Stream m a

673+ [A] enumerate :: (Monad m, Bounded a, Enumerable a) => Stream m a

674+ [A] elemIndices :: (Monad m, Eq a) => a -> Stream m a -> Stream m Int

675+ [A] elem :: (Monad m, Eq a) => a -> Stream m a -> m Bool

676+ [A] durations :: Double -> t m RelTime64

677+ [A] dropWhileM :: Monad m => (a -> m Bool) -> Stream m a -> Stream m a

678+ [A] dropWhileLast :: (a -> Bool) -> Stream m a -> Stream m a

679+ [A] dropWhileAround :: (a -> Bool) -> Stream m a -> Stream m a

680+ [A] dropWhile :: Monad m => (a -> Bool) -> Stream m a -> Stream m a

681+ [A] dropSuffix :: Stream m a -> Stream m a -> Stream m a

682+ [A] dropPrefix :: Stream m a -> Stream m a -> Stream m a

683+ [A] dropLast :: Int -> Stream m a -> Stream m a

684+ [A] dropInfix :: Stream m a -> Stream m a -> Stream m a

685+ [A] drop :: Monad m => Int -> Stream m a -> Stream m a

686+ [A] drain :: Monad m => Stream m a -> m ()

687+ [A] deleteInStreamGenericBy :: Monad m => (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a

688+ [A] deleteInStreamAscBy :: (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a

689+ [A] deleteBy :: Monad m => (a -> a -> Bool) -> a -> Stream m a -> Stream m a

690+ [A] delayPre :: MonadIO m => Double -> Stream m a -> Stream m a

691+ [A] delayPost :: MonadIO m => Double -> Stream m a -> Stream m a

692+ [A] delay :: MonadIO m => Double -> Stream m a -> Stream m a

693+ [A] crossWith :: Monad m => (a -> b -> c) -> Stream m a -> Stream m b -> Stream m c

694+ [A] crossApplySnd :: Functor f => Stream f a -> Stream f b -> Stream f b

695+ [A] crossApplyFst :: Functor f => Stream f a -> Stream f b -> Stream f a

696+ [A] crossApply :: Functor f => Stream f (a -> b) -> Stream f a -> Stream f b

697+ [A] cross :: Monad m => Stream m a -> Stream m b -> Stream m (a, b)

698+ [A] consM :: Applicative m => m a -> Stream m a -> Stream m a

699+ [A] cons :: Applicative m => a -> Stream m a -> Stream m a

700+ [A] concatMapM :: Monad m => (a -> m (Stream m b)) -> Stream m a -> Stream m b

701+ [A] concatMap :: Monad m => (a -> Stream m b) -> Stream m a -> Stream m b

702+ [A] concatIterateScan :: Monad m => (b -> a -> m b) -> (b -> m (Maybe (b, Stream m a))) -> b -> Stream m a

703+ [A] concatIterateDfs :: Monad m => (a -> Maybe (Stream m a)) -> Stream m a -> Stream m a

704+ [A] concatIterateBfsRev :: Monad m => (a -> Maybe (Stream m a)) -> Stream m a -> Stream m a

705+ [A] concatIterateBfs :: Monad m => (a -> Maybe (Stream m a)) -> Stream m a -> Stream m a

706+ [A] concatEffect :: Monad m => m (Stream m a) -> Stream m a

707+ [A] concat :: Monad m => Stream m (Stream m a) -> Stream m a

708+ [A] cmpBy :: Monad m => (a -> b -> Ordering) -> Stream m a -> Stream m b -> m Ordering

709+ [A] catRights :: Monad m => Stream m (Either a b) -> Stream m b

710+ [A] catMaybes :: Monad m => Stream m (Maybe a) -> Stream m a

711+ [A] catLefts :: Monad m => Stream m (Either a b) -> Stream m a

712+ [A] catEithers :: Monad m => Stream m (Either a a) -> Stream m a

713+ [A] bracketUnsafe :: MonadCatch m => m b -> (b -> m c) -> (b -> Stream m a) -> Stream m a

714+ [A] bracketIO3 :: (MonadIO m, MonadCatch m) => IO b -> (b -> IO c) -> (b -> IO d) -> (b -> IO e) -> (b -> Stream m a) -> Stream m a

715+ [A] bracketIO :: (MonadIO m, MonadCatch m) => IO b -> (b -> IO c) -> (b -> Stream m a) -> Stream m a

716+ [A] before :: Monad m => m b -> Stream m a -> Stream m a

717+ [A] append :: Monad m => Stream m a -> Stream m a -> Stream m a

718+ [A] any :: Monad m => (a -> Bool) -> Stream m a -> m Bool

719+ [A] all :: Monad m => (a -> Bool) -> Stream m a -> m Bool

720+ [A] afterUnsafe :: Monad m => m b -> Stream m a -> Stream m a

721+ [A] afterIO :: MonadIO m => IO b -> Stream m a -> Stream m a

722+ [A] absTimesWith :: MonadIO m => Double -> Stream m AbsTime

723+ [A] absTimes :: MonadIO m => Stream m AbsTime

724+ [A] (!!) :: Monad m => Stream m a -> Int -> m (Maybe a)

725+[R] Streamly.Internal.Data.Ring.Unboxed

726+[A] Streamly.Internal.Data.Ring.Generic

727+ [A] Ring

728+ [A] [ringMax] :: Ring a -> !Int

729+ [A] [ringHead] :: Ring a -> !Int

730+ [A] [ringArr] :: Ring a -> MutArray a

731+ [A] Ring :: MutArray a -> !Int -> !Int -> Ring a

732+ [A] writeLastN :: MonadIO m => Int -> Fold m a (Ring a)

733+ [A] unsafeInsertRingWith :: Ring a -> a -> IO Int

734+ [A] toStreamWith :: Int -> Ring a -> Stream m a

735+ [A] toMutArray :: MonadIO m => Int -> Int -> Ring a -> m (MutArray a)

736+ [A] seek :: MonadIO m => Int -> Ring a -> m (Ring a)

737+ [A] createRing :: MonadIO m => Int -> m (Ring a)

738+ [A] copyToMutArray :: MonadIO m => Int -> Int -> Ring a -> m (MutArray a)

739+[C] Streamly.Internal.Data.Ring

740+ [C] Ring

741+ [A] [ringStart] :: Ring a -> {-# UNPACK #-} !ForeignPtr a

742+ [R] [ringMax] :: Ring a -> !Int

743+ [R] [ringHead] :: Ring a -> !Int

744+ [A] [ringBound] :: Ring a -> {-# UNPACK #-} !Ptr a

745+ [R] [ringArr] :: Ring a -> MutArray a

746+ [C] Ring

747+ [O] Ring :: MutArray a -> !Int -> !Int -> Ring a

748+ [N] Ring :: {-# UNPACK #-} !ForeignPtr a -> {-# UNPACK #-} !Ptr a -> Ring a

749+ [A] GHC.Show.Show

750+ [A] instance (GHC.Show.Show a, GHC.Show.Show b, GHC.Show.Show c, GHC.Show.Show d) => GHC.Show.Show (Streamly.Internal.Data.Ring.Tuple4' a b c d)

751+ [A] writeN :: Int -> Fold m a (Ring a)

752+ [R] writeLastN :: MonadIO m => Int -> Fold m a (Ring a)

753+ [R] unsafeInsertRingWith :: Ring a -> a -> IO Int

754+ [A] unsafeInsert :: Storable a => Ring a -> Ptr a -> a -> IO (Ptr a)

755+ [A] unsafeFoldRingNM :: forall m a b. (MonadIO m, Storable a) => Int -> Ptr a -> (b -> a -> m b) -> b -> Ring a -> m b

756+ [A] unsafeFoldRingM :: forall m a b. (MonadIO m, Storable a) => Ptr a -> (b -> a -> m b) -> b -> Ring a -> m b

757+ [A] unsafeFoldRingFullM :: forall m a b. (MonadIO m, Storable a) => Ptr a -> (b -> a -> m b) -> b -> Ring a -> m b

758+ [A] unsafeFoldRing :: forall a b. Storable a => Ptr a -> (b -> a -> b) -> b -> Ring a -> b

759+ [A] unsafeEqArrayN :: Ring a -> Ptr a -> Array a -> Int -> Bool

760+ [A] unsafeEqArray :: Ring a -> Ptr a -> Array a -> Bool

761+ [R] toStreamWith :: Int -> Ring a -> Stream m a

762+ [R] toMutArray :: MonadIO m => Int -> Int -> Ring a -> m (MutArray a)

763+ [A] startOf :: Ring a -> Ptr a

764+ [A] slidingWindowWith :: forall m a b. (MonadIO m, Storable a, Unbox a) => Int -> Fold m ((a, Maybe a), m (MutArray a)) b -> Fold m a b

765+ [A] slidingWindow :: forall m a b. (MonadIO m, Storable a, Unbox a) => Int -> Fold m (a, Maybe a) b -> Fold m a b

766+ [A] slide :: Ring a -> a -> m (Ring a)

767+ [R] seek :: MonadIO m => Int -> Ring a -> m (Ring a)

768+ [A] ringsOf :: Int -> Stream m a -> Stream m (MutArray a)

769+ [A] readRev :: Unfold m (MutArray a) a

770+ [A] read :: forall m a. (MonadIO m, Storable a) => Unfold m (Ring a, Ptr a, Int) a

771+ [A] putIndex :: Ring a -> Int -> a -> m ()

772+ [A] newRing :: Int -> m (Ring a)

773+ [A] new :: forall a. Storable a => Int -> IO (Ring a, Ptr a)

774+ [A] moveBy :: forall a. Storable a => Int -> Ring a -> Ptr a -> Ptr a

775+ [A] modifyIndex :: Ring a -> Int -> (a -> (a, b)) -> m b

776+ [A] length :: Ring a -> Int

777+ [A] getIndexUnsafe :: Ring a -> Int -> m a

778+ [A] getIndexRev :: Ring a -> Int -> m a

779+ [A] getIndex :: Ring a -> Int -> m a

780+ [A] fromArray :: MutArray a -> Ring a

781+ [R] createRing :: MonadIO m => Int -> m (Ring a)

782+ [A] castUnsafe :: Ring a -> Ring b

783+ [A] cast :: forall a b. Storable b => Ring a -> Maybe (Ring b)

784+ [A] bytesFree :: Ring a -> Int

785+ [A] byteLength :: Ring a -> Int

786+ [A] byteCapacity :: Ring a -> Int

787+ [A] asBytes :: Ring a -> Ring Word8

788+ [A] advance :: forall a. Storable a => Ring a -> Ptr a -> Ptr a

789+[R] Streamly.Internal.Data.Producer.Type

790+[R] Streamly.Internal.Data.Producer.Source

791+[C] Streamly.Internal.Data.Producer

792+ [A] Source

793+ [A] unread :: [b] -> Source a b -> Source a b

794+ [A] translate :: Functor m => (a -> c) -> (c -> a) -> Producer m c b -> Producer m a b

795+ [A] source :: Maybe a -> Source a b

796+ [A] producer :: Monad m => Producer m a b -> Producer m (Source a b) b

797+ [A] parseManyD :: Monad m => Parser a m b -> Producer m (Source x a) a -> Producer m (Source x a) (Either ParseError b)

798+ [A] parseMany :: Monad m => Parser a m b -> Producer m (Source x a) a -> Producer m (Source x a) (Either ParseError b)

799+ [A] parse :: Monad m => Parser a m b -> Producer m (Source s a) a -> Source s a -> m (Either ParseError b, Source s a)

800+ [A] lmap :: (a -> a) -> Producer m a b -> Producer m a b

801+ [A] isEmpty :: Source a b -> Bool

802+[R] Streamly.Internal.Data.Pipe.Type

803+[C] Streamly.Internal.Data.Pipe

804+ [A] Step

805+ [A] Yield :: a -> s -> Step s a

806+ [A] Continue :: s -> Step s a

807+ [A] PipeState

808+ [A] Produce :: s2 -> PipeState s1 s2

809+ [A] Consume :: s1 -> PipeState s1 s2

810+ [C] Pipe

811+ [A] Pipe :: (s1 -> a -> m (Step (PipeState s1 s2) b)) -> (s2 -> m (Step (PipeState s1 s2) b)) -> s1 -> Pipe m a b

812+[A] Streamly.Internal.Data.ParserK

813+ [A] Step

814+ [A] Partial :: !Int -> (Input a -> m (Step a m r)) -> Step a m r

815+ [A] Error :: !Int -> String -> Step a m r

816+ [A] Done :: !Int -> r -> Step a m r

817+ [A] Continue :: !Int -> (Input a -> m (Step a m r)) -> Step a m r

818+ [A] ParseResult

819+ [A] Success :: !Int -> !b -> ParseResult b

820+ [A] Failure :: !Int -> !String -> ParseResult b

821+ [A] Input

822+ [A] None :: Input a

823+ [A] Chunk :: a -> Input a

824+ [A] ParserK

825+ [A] [runParser] :: ParserK a m b -> forall r. (ParseResult b -> Int -> Input a -> m (Step a m r)) -> Int -> Int -> Input a -> m (Step a m r)

826+ [A] MkParser :: (forall r. (ParseResult b -> Int -> Input a -> m (Step a m r)) -> Int -> Int -> Input a -> m (Step a m r)) -> ParserK a m b

827+ [A] fromPure :: b -> ParserK a m b

828+ [A] fromEffect :: Monad m => m b -> ParserK a m b

829+ [A] die :: String -> ParserK a m b

830+ [A] adaptCG :: Monad m => Parser a m b -> ParserK (Array a) m b

831+ [A] adaptC :: (Monad m, Unbox a) => Parser a m b -> ParserK (Array a) m b

832+ [A] adapt :: Monad m => Parser a m b -> ParserK a m b

833+[R] Streamly.Internal.Data.Parser.ParserK.Type

834+[R] Streamly.Internal.Data.Parser.ParserD.Type

835+[R] Streamly.Internal.Data.Parser.ParserD.Tee

836+[R] Streamly.Internal.Data.Parser.ParserD

837+[C] Streamly.Internal.Data.Parser

838+ [A] Step

839+ [A] Partial :: !Int -> !s -> Step s b

840+ [A] Error :: !String -> Step s b

841+ [A] Done :: !Int -> !b -> Step s b

842+ [A] Continue :: !Int -> !s -> Step s b

843+ [A] Parser

844+ [A] Parser :: (s -> a -> m (Step s b)) -> m (Initial s b) -> (s -> m (Step s b)) -> Parser a m b

845+ [A] Initial

846+ [A] IPartial :: !s -> Initial s b

847+ [A] IError :: !String -> Initial s b

848+ [A] IDone :: !b -> Initial s b

849+ [A] GHC.Show.Show

850+ [A] instance (GHC.Show.Show a, GHC.Show.Show b) => GHC.Show.Show (Streamly.Internal.Data.Parser.Tuple'Fused a b)

851+ [A] ParseError

852+ [A] ParseError :: String -> ParseError

853+ [A] zipWithM :: Monad m => (a -> b -> m c) -> Stream m a -> Fold m c x -> Parser b m x

854+ [A] zip :: Monad m => Stream m a -> Fold m (a, b) x -> Parser b m x

855+ [A] wordWithQuotes :: (Monad m, Eq a) => Bool -> (a -> a -> Maybe a) -> a -> (a -> Maybe a) -> (a -> Bool) -> Fold m a b -> Parser a m b

856+ [A] wordProcessQuotes :: (Monad m, Eq a) => a -> (a -> Maybe a) -> (a -> Bool) -> Fold m a b -> Parser a m b

857+ [A] wordKeepQuotes :: (Monad m, Eq a) => a -> (a -> Maybe a) -> (a -> Bool) -> Fold m a b -> Parser a m b

858+ [A] wordFramedBy :: Monad m => (a -> Bool) -> (a -> Bool) -> (a -> Bool) -> (a -> Bool) -> Fold m a b -> Parser a m b

859+ [A] wordBy :: Monad m => (a -> Bool) -> Fold m a b -> Parser a m b

860+ [A] toFold :: Monad m => Parser a m b -> Fold m a b

861+ [A] takeWhileP :: Monad m => (a -> Bool) -> Parser a m b -> Parser a m b

862+ [A] takeWhile1 :: Monad m => (a -> Bool) -> Fold m a b -> Parser a m b

863+ [A] takeWhile :: Monad m => (a -> Bool) -> Fold m a b -> Parser a m b

864+ [A] takeStartBy_ :: Monad m => (a -> Bool) -> Fold m a b -> Parser a m b

865+ [A] takeStartBy :: Monad m => (a -> Bool) -> Fold m a b -> Parser a m b

866+ [A] takeP :: Monad m => Int -> Parser a m b -> Parser a m b

867+ [A] takeGE :: Monad m => Int -> Fold m a b -> Parser a m b

868+ [A] takeFramedBy_ :: Monad m => (a -> Bool) -> (a -> Bool) -> Fold m a b -> Parser a m b

869+ [A] takeFramedByGeneric :: Monad m => Maybe (a -> Bool) -> Maybe (a -> Bool) -> Maybe (a -> Bool) -> Fold m a b -> Parser a m b

870+ [A] takeFramedByEsc_ :: Monad m => (a -> Bool) -> (a -> Bool) -> (a -> Bool) -> Fold m a b -> Parser a m b

871+ [A] takeEndBy_ :: (a -> Bool) -> Parser a m b -> Parser a m b

872+ [A] takeEndByEsc :: Monad m => (a -> Bool) -> (a -> Bool) -> Parser a m b -> Parser a m b

873+ [A] takeEndBy :: Monad m => (a -> Bool) -> Parser a m b -> Parser a m b

874+ [A] takeEitherSepBy :: (a -> Bool) -> Fold m (Either a b) c -> Parser a m c

875+ [A] takeEQ :: Monad m => Int -> Fold m a b -> Parser a m b

876+ [A] takeBetween :: Monad m => Int -> Int -> Fold m a b -> Parser a m b

877+ [A] subsequenceBy :: (a -> a -> Bool) -> Stream m a -> Parser a m ()

878+ [A] streamEqBy :: Monad m => (a -> a -> Bool) -> Stream m a -> Parser a m ()

879+ [A] split_ :: Monad m => Parser x m a -> Parser x m b -> Parser x m b

880+ [A] splitWith :: Monad m => (a -> b -> c) -> Parser x m a -> Parser x m b -> Parser x m c

881+ [A] splitSome :: Monad m => Parser a m b -> Fold m b c -> Parser a m c

882+ [A] splitManyPost :: Monad m => Parser a m b -> Fold m b c -> Parser a m c

883+ [A] splitMany :: Monad m => Parser a m b -> Fold m b c -> Parser a m c

884+ [A] spanByRolling :: Monad m => (a -> a -> Bool) -> Fold m a b -> Fold m a c -> Parser a m (b, c)

885+ [A] spanBy :: Monad m => (a -> a -> Bool) -> Fold m a b -> Fold m a c -> Parser a m (b, c)

886+ [A] span :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a c -> Parser a m (b, c)

887+ [A] some :: Monad m => Parser a m b -> Fold m b c -> Parser a m c

888+ [A] sequence :: Monad m => Stream m (Parser a m b) -> Fold m b c -> Parser a m c

889+ [A] sepByAll :: Monad m => Parser a m b -> Parser a m x -> Fold m b c -> Parser a m c

890+ [A] sepBy1 :: Monad m => Parser a m b -> Parser a m x -> Fold m b c -> Parser a m c

891+ [A] sepBy :: Monad m => Parser a m b -> Parser a m x -> Fold m b c -> Parser a m c

892+ [A] satisfy :: Monad m => (a -> Bool) -> Parser a m a

893+ [A] sampleFromthen :: Monad m => Int -> Int -> Fold m a b -> Parser a m b

894+ [A] roundRobin :: t (Parser a m b) -> Fold m b c -> Parser a m c

895+ [A] rmapM :: Monad m => (b -> m c) -> Parser a m b -> Parser a m c

896+ [A] retryMaxTotal :: Int -> Parser a m b -> Fold m b c -> Parser a m c

897+ [A] retryMaxSuccessive :: Int -> Parser a m b -> Fold m b c -> Parser a m c

898+ [A] retry :: Parser a m b -> Parser a m b

899+ [A] postscan :: Fold m a b -> Parser b m c -> Parser a m c

900+ [A] peek :: Monad m => Parser a m a

901+ [A] oneOf :: (Monad m, Eq a, Foldable f) => f a -> Parser a m a

902+ [A] oneNotEq :: (Monad m, Eq a) => a -> Parser a m a

903+ [A] oneEq :: (Monad m, Eq a) => a -> Parser a m a

904+ [A] one :: Monad m => Parser a m a

905+ [A] noneOf :: (Monad m, Eq a, Foldable f) => f a -> Parser a m a

906+ [A] noErrorUnsafeSplit_ :: Monad m => Parser x m a -> Parser x m b -> Parser x m b

907+ [A] noErrorUnsafeSplitWith :: Monad m => (a -> b -> c) -> Parser x m a -> Parser x m b -> Parser x m c

908+ [A] noErrorUnsafeConcatMap :: Monad m => (b -> Parser a m c) -> Parser a m b -> Parser a m c

909+ [A] maybe :: Monad m => (a -> Maybe b) -> Parser a m b

910+ [A] manyTillP :: Parser a m b -> Parser a m x -> Parser b m c -> Parser a m c

911+ [A] manyTill :: Monad m => Parser a m b -> Parser a m x -> Fold m b c -> Parser a m c

912+ [A] manyThen :: Parser a m b -> Parser a m x -> Fold m b c -> Parser a m c

913+ [A] manyP :: Parser a m b -> Parser b m c -> Parser a m c

914+ [A] many :: Monad m => Parser a m b -> Fold m b c -> Parser a m c

915+ [A] makeIndexFilter :: (Fold m (s, a) b -> Parser a m b) -> (((s, a) -> Bool) -> Fold m (s, a) b -> Fold m (s, a) b) -> ((s, a) -> Bool) -> Fold m a b -> Parser a m b

916+ [A] lookAhead :: Monad m => Parser a m b -> Parser a m b

917+ [A] lmapM :: Monad m => (a -> m b) -> Parser b m r -> Parser a m r

918+ [A] lmap :: (a -> b) -> Parser b m r -> Parser a m r

919+ [A] listEqBy :: Monad m => (a -> a -> Bool) -> [a] -> Parser a m [a]

920+ [A] listEq :: (Monad m, Eq a) => [a] -> Parser a m [a]

921+ [A] indexed :: forall m a b. Monad m => Fold m (Int, a) b -> Parser a m b

922+ [A] groupByRollingEither :: Monad m => (a -> a -> Bool) -> Fold m a b -> Fold m a c -> Parser a m (Either b c)

923+ [A] groupByRolling :: Monad m => (a -> a -> Bool) -> Fold m a b -> Parser a m b

924+ [A] groupBy :: Monad m => (a -> a -> Bool) -> Fold m a b -> Parser a m b

925+ [A] fromPure :: Monad m => b -> Parser a m b

926+ [A] fromFoldMaybe :: Monad m => String -> Fold m a (Maybe b) -> Parser a m b

927+ [A] fromFold :: Monad m => Fold m a b -> Parser a m b

928+ [A] fromEffect :: Monad m => m b -> Parser a m b

929+ [A] filter :: Monad m => (a -> Bool) -> Parser a m b -> Parser a m b

930+ [A] extractStep :: Monad m => (s -> m (Step s1 b)) -> Step s b -> m (Step s1 b)

931+ [A] eof :: Monad m => Parser a m ()

932+ [A] either :: Monad m => (a -> Either String b) -> Parser a m b

933+ [A] dropWhile :: Monad m => (a -> Bool) -> Parser a m ()

934+ [A] dieM :: Monad m => m String -> Parser a m b

935+ [A] die :: Monad m => String -> Parser a m b

936+ [A] deintercalateAll :: Monad m => Parser a m x -> Parser a m y -> Fold m (Either x y) z -> Parser a m z

937+ [A] deintercalate1 :: Monad m => Parser a m x -> Parser a m y -> Fold m (Either x y) z -> Parser a m z

938+ [A] deintercalate :: Monad m => Parser a m x -> Parser a m y -> Fold m (Either x y) z -> Parser a m z

939+ [A] countBetween :: Int -> Int -> Parser a m b -> Fold m b c -> Parser a m c

940+ [A] count :: Int -> Parser a m b -> Fold m b c -> Parser a m c

941+ [A] concatMap :: Monad m => (b -> Parser a m c) -> Parser a m b -> Parser a m c

942+ [A] blockWithQuotes :: (Monad m, Eq a) => (a -> Bool) -> (a -> Bool) -> a -> a -> Fold m a b -> Parser a m b

943+ [A] bimapOverrideCount :: Int -> (s -> s1) -> (b -> b1) -> Step s b -> Step s1 b1

944+ [A] alt :: Monad m => Parser x m a -> Parser x m a -> Parser x m a

945+[A] Streamly.Internal.Data.MutByteArray

946+ [A] class Unbox a

947+ [A] class SizeOfRep (f :: Type -> Type)

948+ [A] class Serialize a

949+ [A] TypeOfType

950+ [A] UnitType :: Name -> TypeOfType

951+ [A] TheType :: SimpleDataCon -> TypeOfType

952+ [A] MultiType :: [SimpleDataCon] -> TypeOfType

953+ [A] SimpleDataCon

954+ [A] SimpleDataCon :: Name -> [Field] -> SimpleDataCon

955+ [A] SerializeConfig

956+ [A] [cfgRecordSyntaxWithHeader] :: SerializeConfig -> Bool

957+ [A] [cfgInlineSize] :: SerializeConfig -> Maybe Inline

958+ [A] [cfgInlineSerialize] :: SerializeConfig -> Maybe Inline

959+ [A] [cfgInlineDeserialize] :: SerializeConfig -> Maybe Inline

960+ [A] [cfgConstructorTagAsString] :: SerializeConfig -> Bool

961+ [A] SerializeConfig :: Maybe Inline -> Maybe Inline -> Maybe Inline -> Bool -> Bool -> SerializeConfig

962+ [A] PinnedState

963+ [A] Unpinned :: PinnedState

964+ [A] Pinned :: PinnedState

965+ [A] MutByteArray

966+ [A] MutByteArray :: MutableByteArray# RealWorld -> MutByteArray

967+ [A] DataType

968+ [A] [dtTvs] :: DataType -> [Name]

969+ [A] [dtName] :: DataType -> Name

970+ [A] [dtCxt] :: DataType -> Cxt

971+ [A] [dtCons] :: DataType -> [DataCon]

972+ [A] DataType :: Name -> [Name] -> Cxt -> [DataCon] -> DataType

973+ [A] DataCon

974+ [A] [dcTvs] :: DataCon -> [Name]

975+ [A] [dcName] :: DataCon -> Name

976+ [A] [dcFields] :: DataCon -> [(Maybe Name, Type)]

977+ [A] [dcCxt] :: DataCon -> Cxt

978+ [A] DataCon :: Name -> [Name] -> Cxt -> [(Maybe Name, Type)] -> DataCon

979+ [A] BoundedPtr

980+ [A] BoundedPtr :: MutByteArray -> Int -> Int -> BoundedPtr

981+ [A] Streamly.Internal.Data.Serialize.Type.Serialize

982+ [A] instance Streamly.Internal.Data.Serialize.Type.Serialize Streamly.Internal.Data.MutByteArray.LiftedInteger

983+ [A] instance Streamly.Internal.Data.Serialize.Type.Serialize GHC.Num.Integer.Integer

984+ [A] instance Streamly.Internal.Data.Serialize.Type.Serialize a => Streamly.Internal.Data.Serialize.Type.Serialize (GHC.Maybe.Maybe a)

985+ [A] instance Streamly.Internal.Data.Serialize.Type.Serialize (Data.Proxy.Proxy a)

986+ [A] instance (Streamly.Internal.Data.Serialize.Type.Serialize a, Streamly.Internal.Data.Serialize.Type.Serialize b) => Streamly.Internal.Data.Serialize.Type.Serialize (Data.Either.Either a b)

987+ [A] Peeker

988+ [A] Peeker :: Builder BoundedPtr IO a -> Peeker a

989+ [A] type Field = (Maybe Name, Type)

990+ [A] xorCmp :: [Word8] -> Name -> Name -> Q Exp

991+ [A] wListToString :: [Word8] -> String

992+ [A] w8_int :: Word8 -> Int

993+ [A] w32_int :: Word32 -> Int

994+ [A] unpin :: MutByteArray -> IO MutByteArray

995+ [A] typeOfType :: Type -> [DataCon] -> TypeOfType

996+ [A] skipByte :: Peeker ()

997+ [A] sizeOfRep :: SizeOfRep f => f x -> Int

998+ [A] sizeOfMutableByteArray :: MutByteArray -> IO Int

999+ [A] sizeOf :: (Unbox a, SizeOfRep (Rep a)) => Proxy a -> Int

1000+ [A] simplifyDataCon :: DataCon -> SimpleDataCon

1001+ [A] serializeW8List :: Name -> Name -> [Word8] -> Q Exp

1002+ [A] serializeConfig :: SerializeConfig

1003+ [A] serializeAt :: Serialize a => Int -> MutByteArray -> a -> IO Int

1004+ [A] runPeeker :: Peeker a -> BoundedPtr -> IO a

1005+ [A] reifyDataType :: Name -> Q DataType

1006+ [A] readUnsafe :: Unbox a => Peeker a

1007+ [A] read :: Unbox a => Peeker a

1008+ [A] putSliceUnsafe :: MonadIO m => MutByteArray -> Int -> MutByteArray -> Int -> Int -> m ()

1009+ [A] pokeRep :: PokeRep f => f a -> BoundedPtr -> IO BoundedPtr

1010+ [A] pokeBoundedPtrUnsafe :: forall a. Unbox a => a -> BoundedPtr -> IO BoundedPtr

1011+ [A] pokeBoundedPtr :: forall a. Unbox a => a -> BoundedPtr -> IO BoundedPtr

1012+ [A] pokeAt :: (Unbox a, Generic a, PokeRep (Rep a)) => Int -> MutByteArray -> a -> IO ()

1013+ [A] pinnedNewAlignedBytes :: Int -> Int -> IO MutByteArray

1014+ [A] pinnedNew :: Int -> IO MutByteArray

1015+ [A] pinnedCloneSliceUnsafe :: MonadIO m => Int -> Int -> MutByteArray -> m MutByteArray

1016+ [A] pin :: MutByteArray -> IO MutByteArray

1017+ [A] peekRep :: PeekRep f => Peeker (f x)

1018+ [A] peekAt :: (Unbox a, Generic a, PeekRep (Rep a)) => Int -> MutByteArray -> IO a

1019+ [A] openConstructor :: Name -> Int -> Q Pat

1020+ [A] nil :: MutByteArray

1021+ [A] newBytesAs :: PinnedState -> Int -> IO MutByteArray

1022+ [A] new :: Int -> IO MutByteArray

1023+ [A] mkSerializeExprFields :: Name -> [Field] -> Q Exp

1024+ [A] mkRecSizeOfExpr :: SimpleDataCon -> Q Exp

1025+ [A] mkRecSerializeExpr :: Name -> SimpleDataCon -> Q Exp

1026+ [A] mkRecDeserializeExpr :: Name -> Name -> Name -> SimpleDataCon -> Q Exp

1027+ [A] mkFieldName :: Int -> Name

1028+ [A] mkDeserializeKeysDec :: Name -> Name -> SimpleDataCon -> Q [Dec]

1029+ [A] mkDeserializeExprOne :: Name -> SimpleDataCon -> Q Exp

1030+ [A] matchConstructor :: Name -> Int -> Q Exp -> Q Match

1031+ [A] makeN :: Int -> Name

1032+ [A] makeI :: Int -> Name

1033+ [A] makeA :: Int -> Name

1034+ [A] litProxy :: Unbox a => Proxy a -> Q Exp

1035+ [A] litIntegral :: Integral a => a -> Q Exp

1036+ [A] isUnitType :: [DataCon] -> Bool

1037+ [A] isRecordSyntax :: SimpleDataCon -> Bool

1038+ [A] isPinned :: MutByteArray -> Bool

1039+ [A] int_w8 :: Int -> Word8

1040+ [A] int_w32 :: Int -> Word32

1041+ [A] inlineSerializeAt :: Maybe Inline -> SerializeConfig -> SerializeConfig

1042+ [A] inlineDeserializeAt :: Maybe Inline -> SerializeConfig -> SerializeConfig

1043+ [A] inlineAddSizeTo :: Maybe Inline -> SerializeConfig -> SerializeConfig

1044+ [A] getMutableByteArray# :: MutByteArray -> MutableByteArray# RealWorld

1045+ [A] genericSizeOf :: forall a. SizeOfRep (Rep a) => Proxy a -> Int

1046+ [A] genericPokeByteIndex :: (Generic a, PokeRep (Rep a)) => MutByteArray -> Int -> a -> IO ()

1047+ [A] genericPeekByteIndex :: (Generic a, PeekRep (Rep a)) => MutByteArray -> Int -> IO a

1048+ [A] errorUnsupported :: String -> a

1049+ [A] errorUnimplemented :: a

1050+ [A] encodeRecordFields :: Bool -> SerializeConfig -> SerializeConfig

1051+ [A] encodeConstrNames :: Bool -> SerializeConfig -> SerializeConfig

1052+ [A] deserializeAt :: Serialize a => Int -> MutByteArray -> Int -> IO (Int, a)

1053+ [A] deriveUnbox :: Q [Dec] -> Q [Dec]

1054+ [A] deriveSerializeWith :: (SerializeConfig -> SerializeConfig) -> Q [Dec] -> Q [Dec]

1055+ [A] deriveSerialize :: Q [Dec] -> Q [Dec]

1056+ [A] conUpdateFuncDec :: Name -> [Field] -> Q [Dec]

1057+ [A] cloneSliceUnsafeAs :: MonadIO m => PinnedState -> Int -> Int -> MutByteArray -> m MutByteArray

1058+ [A] cloneSliceUnsafe :: MonadIO m => Int -> Int -> MutByteArray -> m MutByteArray

1059+ [A] c2w :: Char -> Word8

1060+ [A] asPtrUnsafe :: MonadIO m => MutByteArray -> (Ptr a -> m b) -> m b

1061+ [A] addSizeTo :: Serialize a => Int -> a -> Int

1062+ [A] _x :: Name

1063+ [A] _val :: Name

1064+ [A] _tag :: Name

1065+ [A] _initialOffset :: Name

1066+ [A] _endOffset :: Name

1067+ [A] _arr :: Name

1068+ [A] _acc :: Name

1069+[A] Streamly.Internal.Data.MutArray.Stream

1070+ [A] SpliceState

1071+ [A] SpliceYielding :: arr -> SpliceState s arr -> SpliceState s arr

1072+ [A] SpliceInitial :: s -> SpliceState s arr

1073+ [A] SpliceFinish :: SpliceState s arr

1074+ [A] SpliceBuffering :: s -> arr -> SpliceState s arr

1075+ [A] writeChunks :: (MonadIO m, Unbox a) => Int -> Fold m a (StreamK n (MutArray a))

1076+ [A] splitOn :: (MonadIO m, Unbox a) => (a -> Bool) -> MutArray a -> Stream m (MutArray a)

1077+ [A] pinnedChunksOf :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> Stream m (MutArray a)

1078+ [A] packArraysChunksOf :: (MonadIO m, Unbox a) => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

1079+ [A] lpackArraysChunksOf :: (MonadIO m, Unbox a) => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()

1080+ [A] fromArrayStreamK :: (Unbox a, MonadIO m) => StreamK m (MutArray a) -> m (MutArray a)

1081+ [A] flattenArraysRev :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> Stream m a

1082+ [A] flattenArrays :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> Stream m a

1083+ [A] compactLE :: (MonadIO m, Unbox a) => Int -> Stream m (MutArray a) -> Stream m (Either ParseError (MutArray a))

1084+ [A] compactGE :: (MonadIO m, Unbox a) => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

1085+ [A] compactEQ :: Int -> Stream m (MutArray a) -> Stream m (MutArray a)

1086+ [A] compact :: (MonadIO m, Unbox a) => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

1087+ [A] chunksOf :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> Stream m (MutArray a)

1088+[A] Streamly.Internal.Data.MutArray.Generic

1089+ [A] MutArray

1090+ [A] [arrTrueLen] :: MutArray a -> {-# UNPACK #-} !Int

1091+ [A] [arrStart] :: MutArray a -> {-# UNPACK #-} !Int

1092+ [A] [arrLen] :: MutArray a -> {-# UNPACK #-} !Int

1093+ [A] [arrContents#] :: MutArray a -> MutableArray# RealWorld a

1094+ [A] MutArray :: MutableArray# RealWorld a -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> MutArray a

1095+ [A] writeWith :: MonadIO m => Int -> Fold m a (MutArray a)

1096+ [A] writeNUnsafe :: MonadIO m => Int -> Fold m a (MutArray a)

1097+ [A] writeN :: MonadIO m => Int -> Fold m a (MutArray a)

1098+ [A] write :: MonadIO m => Fold m a (MutArray a)

1099+ [A] uninit :: MonadIO m => MutArray a -> Int -> m (MutArray a)

1100+ [A] toStreamK :: MonadIO m => MutArray a -> StreamK m a

1101+ [A] toList :: MonadIO m => MutArray a -> m [a]

1102+ [A] strip :: MonadIO m => (a -> Bool) -> MutArray a -> m (MutArray a)

1103+ [A] snocWith :: MonadIO m => (Int -> Int) -> MutArray a -> a -> m (MutArray a)

1104+ [A] snocUnsafe :: MonadIO m => MutArray a -> a -> m (MutArray a)

1105+ [A] snoc :: MonadIO m => MutArray a -> a -> m (MutArray a)

1106+ [A] realloc :: MonadIO m => Int -> MutArray a -> m (MutArray a)

1107+ [A] reader :: MonadIO m => Unfold m (MutArray a) a

1108+ [A] readRev :: MonadIO m => MutArray a -> Stream m a

1109+ [A] read :: MonadIO m => MutArray a -> Stream m a

1110+ [A] putSliceUnsafe :: MonadIO m => MutArray a -> Int -> MutArray a -> Int -> Int -> m ()

1111+ [A] putIndices :: MonadIO m => MutArray a -> Fold m (Int, a) ()

1112+ [A] putIndexUnsafe :: forall m a. MonadIO m => Int -> MutArray a -> a -> m ()

1113+ [A] putIndex :: MonadIO m => Int -> MutArray a -> a -> m ()

1114+ [A] producerWith :: Monad m => (forall b. IO b -> m b) -> Producer m (MutArray a) a

1115+ [A] producer :: MonadIO m => Producer m (MutArray a) a

1116+ [A] nil :: MonadIO m => m (MutArray a)

1117+ [A] new :: MonadIO m => Int -> m (MutArray a)

1118+ [A] modifyIndexUnsafe :: MonadIO m => Int -> MutArray a -> (a -> (a, b)) -> m b

1119+ [A] modifyIndex :: MonadIO m => Int -> MutArray a -> (a -> (a, b)) -> m b

1120+ [A] length :: MutArray a -> Int

1121+ [A] getSliceUnsafe :: Int -> Int -> MutArray a -> MutArray a

1122+ [A] getSlice :: Int -> Int -> MutArray a -> MutArray a

1123+ [A] getIndexUnsafeWith :: MonadIO m => MutableArray# RealWorld a -> Int -> m a

1124+ [A] getIndexUnsafe :: MonadIO m => Int -> MutArray a -> m a

1125+ [A] getIndex :: MonadIO m => Int -> MutArray a -> m (Maybe a)

1126+ [A] fromStreamN :: MonadIO m => Int -> Stream m a -> m (MutArray a)

1127+ [A] fromStream :: MonadIO m => Stream m a -> m (MutArray a)

1128+ [A] fromPureStream :: MonadIO m => Stream Identity a -> m (MutArray a)

1129+ [A] fromListN :: MonadIO m => Int -> [a] -> m (MutArray a)

1130+ [A] fromList :: MonadIO m => [a] -> m (MutArray a)

1131+ [A] eq :: (MonadIO m, Eq a) => MutArray a -> MutArray a -> m Bool

1132+ [A] cmp :: (MonadIO m, Ord a) => MutArray a -> MutArray a -> m Ordering

1133+ [A] clone :: MonadIO m => MutArray a -> m (MutArray a)

1134+ [A] chunksOf :: forall m a. MonadIO m => Int -> Stream m a -> Stream m (MutArray a)

1135+[A] Streamly.Internal.Data.MutArray

1136+ [A] MutByteArray

1137+ [A] MutArray

1138+ [A] [arrStart] :: MutArray a -> {-# UNPACK #-} !Int

1139+ [A] [arrEnd] :: MutArray a -> {-# UNPACK #-} !Int

1140+ [A] [arrContents] :: MutArray a -> {-# UNPACK #-} !MutByteArray

1141+ [A] [arrBound] :: MutArray a -> {-# UNPACK #-} !Int

1142+ [A] MutArray :: {-# UNPACK #-} !MutByteArray -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> MutArray a

1143+ [A] IORef

1144+ [A] ArrayUnsafe

1145+ [A] ArrayUnsafe :: {-# UNPACK #-} !MutByteArray -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> ArrayUnsafe a

1146+ [A] writeWith :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

1147+ [A] writeRevN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

1148+ [A] writeNWithUnsafe :: forall m a. (MonadIO m, Unbox a) => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)

1149+ [A] writeNWith :: forall m a. (MonadIO m, Unbox a) => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)

1150+ [A] writeNUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

1151+ [A] writeN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

1152+ [A] writeIORef :: Unbox a => IORef a -> a -> IO ()

1153+ [A] writeChunks :: (MonadIO m, Unbox a) => Int -> Fold m a (StreamK n (MutArray a))

1154+ [A] writeAppendWith :: forall m a. (MonadIO m, Unbox a) => (Int -> Int) -> m (MutArray a) -> Fold m a (MutArray a)

1155+ [A] writeAppendNUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a) -> Fold m a (MutArray a)

1156+ [A] writeAppendN :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a) -> Fold m a (MutArray a)

1157+ [A] writeAppend :: forall m a. (MonadIO m, Unbox a) => m (MutArray a) -> Fold m a (MutArray a)

1158+ [A] write :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)

1159+ [A] unsafeSwapIndices :: forall m a. (MonadIO m, Unbox a) => Int -> Int -> MutArray a -> m ()

1160+ [A] unpin :: MutArray a -> IO (MutArray a)

1161+ [A] toStreamKWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> MutArray a -> StreamK m a

1162+ [A] toStreamKRevWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> MutArray a -> StreamK m a

1163+ [A] toStreamKRev :: forall m a. (MonadIO m, Unbox a) => MutArray a -> StreamK m a

1164+ [A] toStreamK :: forall m a. (MonadIO m, Unbox a) => MutArray a -> StreamK m a

1165+ [A] toStreamDWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> MutArray a -> Stream m a

1166+ [A] toStreamDRevWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> MutArray a -> Stream m a

1167+ [A] toList :: forall m a. (MonadIO m, Unbox a) => MutArray a -> m [a]

1168+ [A] swapIndices :: forall m a. (MonadIO m, Unbox a) => Int -> Int -> MutArray a -> m ()

1169+ [A] strip :: forall a m. (Unbox a, MonadIO m) => (a -> Bool) -> MutArray a -> m (MutArray a)

1170+ [A] splitOn :: (MonadIO m, Unbox a) => (a -> Bool) -> MutArray a -> Stream m (MutArray a)

1171+ [A] splitAt :: forall a. Unbox a => Int -> MutArray a -> (MutArray a, MutArray a)

1172+ [A] spliceWith :: forall m a. (MonadIO m, Unbox a) => (Int -> Int -> Int) -> MutArray a -> MutArray a -> m (MutArray a)

1173+ [A] spliceUnsafe :: MonadIO m => MutArray a -> MutArray a -> m (MutArray a)

1174+ [A] spliceExp :: (MonadIO m, Unbox a) => MutArray a -> MutArray a -> m (MutArray a)

1175+ [A] spliceCopy :: forall m a. MonadIO m => MutArray a -> MutArray a -> m (MutArray a)

1176+ [A] splice :: (MonadIO m, Unbox a) => MutArray a -> MutArray a -> m (MutArray a)

1177+ [A] snocWith :: forall m a. (MonadIO m, Unbox a) => (Int -> Int) -> MutArray a -> a -> m (MutArray a)

1178+ [A] snocUnsafe :: forall m a. (MonadIO m, Unbox a) => MutArray a -> a -> m (MutArray a)

1179+ [A] snocMay :: forall m a. (MonadIO m, Unbox a) => MutArray a -> a -> m (Maybe (MutArray a))

1180+ [A] snocLinear :: forall m a. (MonadIO m, Unbox a) => MutArray a -> a -> m (MutArray a)

1181+ [A] snoc :: forall m a. (MonadIO m, Unbox a) => MutArray a -> a -> m (MutArray a)

1182+ [A] shuffleBy :: (a -> a -> m Bool) -> MutArray a -> MutArray a -> m ()

1183+ [A] roundUpToPower2 :: Int -> Int

1184+ [A] rightSize :: forall m a. (MonadIO m, Unbox a) => MutArray a -> m (MutArray a)

1185+ [A] reverse :: forall m a. (MonadIO m, Unbox a) => MutArray a -> m ()

1186+ [A] resizeExp :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)

1187+ [A] resize :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)

1188+ [A] realloc :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)

1189+ [A] readerRevWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> Unfold m (MutArray a) a

1190+ [A] readerRev :: forall m a. (MonadIO m, Unbox a) => Unfold m (MutArray a) a

1191+ [A] reader :: forall m a. (MonadIO m, Unbox a) => Unfold m (MutArray a) a

1192+ [A] readRev :: forall m a. (MonadIO m, Unbox a) => MutArray a -> Stream m a

1193+ [A] readIORef :: Unbox a => IORef a -> IO a

1194+ [A] read :: forall m a. (MonadIO m, Unbox a) => MutArray a -> Stream m a

1195+ [A] putIndices :: forall m a. (MonadIO m, Unbox a) => MutArray a -> Fold m (Int, a) ()

1196+ [A] putIndexUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> a -> m ()

1197+ [A] putIndex :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> a -> m ()

1198+ [A] producerWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> Producer m (MutArray a) a

1199+ [A] producer :: forall m a. (MonadIO m, Unbox a) => Producer m (MutArray a) a

1200+ [A] pollIntIORef :: (MonadIO m, Unbox a) => IORef a -> Stream m a

1201+ [A] pinnedWriteNUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

1202+ [A] pinnedWriteNAligned :: forall m a. (MonadIO m, Unbox a) => Int -> Int -> Fold m a (MutArray a)

1203+ [A] pinnedWriteN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

1204+ [A] pinnedWrite :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)

1205+ [A] pinnedNewBytes :: MonadIO m => Int -> m (MutArray a)

1206+ [A] pinnedNewAligned :: (MonadIO m, Unbox a) => Int -> Int -> m (MutArray a)

1207+ [A] pinnedNew :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a)

1208+ [A] pinnedFromListN :: (MonadIO m, Unbox a) => Int -> [a] -> m (MutArray a)

1209+ [A] pinnedFromList :: (MonadIO m, Unbox a) => [a] -> m (MutArray a)

1210+ [A] pinnedClone :: MonadIO m => MutArray a -> m (MutArray a)

1211+ [A] pinnedChunksOf :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> Stream m (MutArray a)

1212+ [A] pin :: MutArray a -> IO (MutArray a)

1213+ [A] permute :: MutArray a -> m Bool

1214+ [A] partitionBy :: forall m a. (MonadIO m, Unbox a) => (a -> Bool) -> MutArray a -> m (MutArray a, MutArray a)

1215+ [A] nil :: MutArray a

1216+ [A] newIORef :: forall a. Unbox a => a -> IO (IORef a)

1217+ [A] newArrayWith :: forall m a. (MonadIO m, Unbox a) => (Int -> Int -> m MutByteArray) -> Int -> Int -> m (MutArray a)

1218+ [A] new :: (MonadIO m, Unbox a) => Int -> m (MutArray a)

1219+ [A] modifyIndices :: forall m a. (MonadIO m, Unbox a) => MutArray a -> (Int -> a -> a) -> Fold m Int ()

1220+ [A] modifyIndexUnsafe :: forall m a b. (MonadIO m, Unbox a) => Int -> MutArray a -> (a -> (a, b)) -> m b

1221+ [A] modifyIndex :: forall m a b. (MonadIO m, Unbox a) => Int -> MutArray a -> (a -> (a, b)) -> m b

1222+ [A] modifyIORef' :: Unbox a => IORef a -> (a -> a) -> IO ()

1223+ [A] modify :: forall m a. (MonadIO m, Unbox a) => MutArray a -> (a -> a) -> m ()

1224+ [A] mergeBy :: Int -> (MutArray a -> MutArray a -> m ()) -> MutArray a -> m ()

1225+ [A] memcpy :: Ptr Word8 -> Ptr Word8 -> Int -> IO ()

1226+ [A] memcmp :: Ptr Word8 -> Ptr Word8 -> Int -> IO Bool

1227+ [A] length :: forall a. Unbox a => MutArray a -> Int

1228+ [A] isPinned :: MutArray a -> Bool

1229+ [A] getSlicesFromLen :: forall m a. (Monad m, Unbox a) => Int -> Int -> Unfold m (MutArray a) (MutArray a)

1230+ [A] getSliceUnsafe :: forall a. Unbox a => Int -> Int -> MutArray a -> MutArray a

1231+ [A] getSlice :: forall a. Unbox a => Int -> Int -> MutArray a -> MutArray a

1232+ [A] getIndicesD :: (Monad m, Unbox a) => (forall b. IO b -> m b) -> Stream m Int -> Unfold m (MutArray a) a

1233+ [A] getIndices :: (MonadIO m, Unbox a) => Stream m Int -> Unfold m (MutArray a) a

1234+ [A] getIndexUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m a

1235+ [A] getIndexRev :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m a

1236+ [A] getIndex :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (Maybe a)

1237+ [A] genSlicesFromLen :: forall m a. (Monad m, Unbox a) => Int -> Int -> Unfold m (MutArray a) (Int, Int)

1238+ [A] fromStreamDN :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> m (MutArray a)

1239+ [A] fromStreamD :: (MonadIO m, Unbox a) => Stream m a -> m (MutArray a)

1240+ [A] fromStream :: (MonadIO m, Unbox a) => Stream m a -> m (MutArray a)

1241+ [A] fromPureStream :: (MonadIO m, Unbox a) => Stream Identity a -> m (MutArray a)

1242+ [A] fromListRevN :: (MonadIO m, Unbox a) => Int -> [a] -> m (MutArray a)

1243+ [A] fromListRev :: (MonadIO m, Unbox a) => [a] -> m (MutArray a)

1244+ [A] fromListN :: (MonadIO m, Unbox a) => Int -> [a] -> m (MutArray a)

1245+ [A] fromList :: (MonadIO m, Unbox a) => [a] -> m (MutArray a)

1246+ [A] fromArrayStreamK :: (Unbox a, MonadIO m) => StreamK m (MutArray a) -> m (MutArray a)

1247+ [A] foldr :: (MonadIO m, Unbox a) => (a -> b -> b) -> b -> MutArray a -> m b

1248+ [A] foldl' :: (MonadIO m, Unbox a) => (b -> a -> b) -> b -> MutArray a -> m b

1249+ [A] flattenArraysRev :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> Stream m a

1250+ [A] flattenArrays :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> Stream m a

1251+ [A] divideBy :: Int -> (MutArray a -> m (MutArray a, MutArray a)) -> MutArray a -> m ()

1252+ [A] cmp :: MonadIO m => MutArray a -> MutArray a -> m Ordering

1253+ [A] clone :: MonadIO m => MutArray a -> m (MutArray a)

1254+ [A] chunksOf :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> Stream m (MutArray a)

1255+ [A] castUnsafe :: MutArray a -> MutArray b

1256+ [A] cast :: forall a b. Unbox b => MutArray a -> Maybe (MutArray b)

1257+ [A] c_memchr :: Ptr Word8 -> Word8 -> CSize -> IO (Ptr Word8)

1258+ [A] bytesFree :: MutArray a -> Int

1259+ [A] byteLength :: MutArray a -> Int

1260+ [A] byteCapacity :: MutArray a -> Int

1261+ [A] bubble :: (MonadIO m, Unbox a) => (a -> a -> Ordering) -> MutArray a -> m ()

1262+ [A] breakOn :: MonadIO m => Word8 -> MutArray Word8 -> m (MutArray Word8, Maybe (MutArray Word8))

1263+ [A] blockSize :: Int

1264+ [A] asPtrUnsafe :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b

1265+ [A] asBytes :: MutArray a -> MutArray Word8

1266+ [A] arrayChunkBytes :: Int

1267+ [A] allocBytesToElemCount :: Unbox a => a -> Int -> Int

1268+[C] Streamly.Internal.Data.IsMap

1269+ [A] mapTraverseWithKey :: (IsMap f, Applicative t) => (Key f -> a -> t b) -> f a -> t (f b)

1270+[R] Streamly.Internal.Data.IORef.Unboxed

1271+[R] Streamly.Internal.Data.Fold.Window

1272+[R] Streamly.Internal.Data.Fold.Type

1273+[R] Streamly.Internal.Data.Fold.Tee

1274+[R] Streamly.Internal.Data.Fold.Step

1275+[R] Streamly.Internal.Data.Fold.Container

1276+[D] Streamly.Internal.Data.Fold.Chunked

1277+[C] Streamly.Internal.Data.Fold

1278+ [A] ManyState

1279+ [C] Fold

1280+ [C] Fold

1281+ [O] Fold :: (s -> a -> m (Step s b)) -> m (Step s b) -> (s -> m b) -> Fold m a b

1282+ [N] Fold :: (s -> a -> m (Step s b)) -> m (Step s b) -> (s -> m b) -> (s -> m b) -> Fold m a b

1283+ [A] windowSumInt :: forall m a. (Monad m, Integral a) => Fold m (a, Maybe a) a

1284+ [A] windowSum :: forall m a. (Monad m, Num a) => Fold m (a, Maybe a) a

1285+ [A] windowRollingMapM :: Monad m => (Maybe a -> a -> m (Maybe b)) -> Fold m (a, Maybe a) (Maybe b)

1286+ [A] windowRollingMap :: Monad m => (Maybe a -> a -> Maybe b) -> Fold m (a, Maybe a) (Maybe b)

1287+ [A] windowRange :: (MonadIO m, Storable a, Ord a) => Int -> Fold m a (Maybe (a, a))

1288+ [A] windowPowerSumFrac :: (Monad m, Floating a) => a -> Fold m (a, Maybe a) a

1289+ [A] windowPowerSum :: (Monad m, Num a) => Int -> Fold m (a, Maybe a) a

1290+ [A] windowMinimum :: (MonadIO m, Storable a, Ord a) => Int -> Fold m a (Maybe a)

1291+ [A] windowMean :: forall m a. (Monad m, Fractional a) => Fold m (a, Maybe a) a

1292+ [A] windowMaximum :: (MonadIO m, Storable a, Ord a) => Int -> Fold m a (Maybe a)

1293+ [A] windowLmap :: (c -> a) -> Fold m (a, Maybe a) b -> Fold m (c, Maybe c) b

1294+ [A] windowLength :: (Monad m, Num b) => Fold m (a, Maybe a) b

1295+ [A] toSet :: (Monad m, Ord a) => Fold m a (Set a)

1296+ [A] toMapIO :: (MonadIO m, Ord k) => (a -> k) -> Fold m a b -> Fold m a (Map k b)

1297+ [A] toMap :: (Monad m, Ord k) => (a -> k) -> Fold m a b -> Fold m a (Map k b)

1298+ [A] toIntSet :: Monad m => Fold m Int IntSet

1299+ [A] toContainerIO :: (MonadIO m, IsMap f, Traversable f, Ord (Key f)) => (a -> Key f) -> Fold m a b -> Fold m a (f b)

1300+ [A] toContainer :: (Monad m, IsMap f, Traversable f, Ord (Key f)) => (a -> Key f) -> Fold m a b -> Fold m a (f b)

1301+ [A] nubInt :: Monad m => Fold m Int (Maybe Int)

1302+ [A] nub :: (Monad m, Ord a) => Fold m a (Maybe a)

1303+ [A] mapMStep :: Applicative m => (a -> m b) -> Step s a -> m (Step s b)

1304+ [A] kvToMap :: (Monad m, Ord k) => Fold m a b -> Fold m (k, a) (Map k b)

1305+ [A] frequency :: (Monad m, Ord a) => Fold m a (Map a Int)

1306+ [A] demuxToMapIO :: (MonadIO m, Ord k) => (a -> k) -> (a -> m (Fold m a b)) -> Fold m a (Map k b)

1307+ [A] demuxToMap :: (Monad m, Ord k) => (a -> k) -> (a -> m (Fold m a b)) -> Fold m a (Map k b)

1308+ [A] demuxToContainerIO :: (MonadIO m, IsMap f, Traversable f) => (a -> Key f) -> (a -> m (Fold m a b)) -> Fold m a (f b)

1309+ [A] demuxToContainer :: (Monad m, IsMap f, Traversable f) => (a -> Key f) -> (a -> m (Fold m a b)) -> Fold m a (f b)

1310+ [A] demuxKvToMap :: (Monad m, Ord k) => (k -> m (Fold m a b)) -> Fold m (k, a) (Map k b)

1311+ [A] demuxKvToContainer :: (Monad m, IsMap f, Traversable f) => (Key f -> m (Fold m a b)) -> Fold m (Key f, a) (f b)

1312+ [A] demuxIO :: (MonadIO m, Ord k) => (a -> k) -> (a -> m (Fold m a b)) -> Fold m a (m (Map k b), Maybe (k, b))

1313+ [A] demuxGenericIO :: (MonadIO m, IsMap f, Traversable f) => (a -> Key f) -> (a -> m (Fold m a b)) -> Fold m a (m (f b), Maybe (Key f, b))

1314+ [A] demuxGeneric :: (Monad m, IsMap f, Traversable f) => (a -> Key f) -> (a -> m (Fold m a b)) -> Fold m a (m (f b), Maybe (Key f, b))

1315+ [A] demux :: (Monad m, Ord k) => (a -> k) -> (a -> m (Fold m a b)) -> Fold m a (m (Map k b), Maybe (k, b))

1316+ [A] cumulative :: Fold m (a, Maybe a) b -> Fold m a b

1317+ [A] countDistinctInt :: Monad m => Fold m Int Int

1318+ [A] countDistinct :: (Monad m, Ord a) => Fold m a Int

1319+ [A] classifyIO :: (MonadIO m, Ord k) => (a -> k) -> Fold m a b -> Fold m a (m (Map k b), Maybe (k, b))

1320+ [A] classifyGenericIO :: (MonadIO m, IsMap f, Traversable f, Ord (Key f)) => (a -> Key f) -> Fold m a b -> Fold m a (m (f b), Maybe (Key f, b))

1321+ [A] classifyGeneric :: (Monad m, IsMap f, Traversable f, Ord (Key f)) => (a -> Key f) -> Fold m a b -> Fold m a (m (f b), Maybe (Key f, b))

1322+ [A] classify :: (Monad m, Ord k) => (a -> k) -> Fold m a b -> Fold m a (m (Map k b), Maybe (k, b))

1323+ [A] chainStepM :: Applicative m => (s1 -> m s2) -> (a -> m (Step s2 b)) -> Step s1 a -> m (Step s2 b)

1324+[C] Streamly.Internal.Data.Builder

1325+ [C] Builder

1326+ [C] Builder

1327+ [O] Builder :: (s -> m (s, a)) -> Builder s m a

1328+ [N] Builder :: (s -> m (a, s)) -> Builder s m a

1329+[A] Streamly.Internal.Data.Binary.Stream

1330+ [A] class ToBytes a

1331+ [A] word8 :: Applicative m => Word8 -> Stream m Word8

1332+ [A] word64le :: Monad m => Word64 -> Stream m Word8

1333+ [A] word64host :: Monad m => Word64 -> Stream m Word8

1334+ [A] word64be :: Monad m => Word64 -> Stream m Word8

1335+ [A] word32le :: Monad m => Word32 -> Stream m Word8

1336+ [A] word32be :: Monad m => Word32 -> Stream m Word8

1337+ [A] word16le :: Monad m => Word16 -> Stream m Word8

1338+ [A] word16be :: Monad m => Word16 -> Stream m Word8

1339+ [A] unit :: Applicative m => Stream m Word8

1340+ [A] toBytes :: ToBytes a => a -> Stream m Word8

1341+ [A] ordering :: Applicative m => Ordering -> Stream m Word8

1342+ [A] int8 :: Applicative m => Int8 -> Stream m Word8

1343+ [A] int64le :: Monad m => Int64 -> Stream m Word8

1344+ [A] int64be :: Monad m => Int64 -> Stream m Word8

1345+ [A] int32le :: Monad m => Int32 -> Stream m Word8

1346+ [A] int32be :: Monad m => Int32 -> Stream m Word8

1347+ [A] int16le :: Monad m => Int16 -> Stream m Word8

1348+ [A] int16be :: Monad m => Int16 -> Stream m Word8

1349+ [A] float32le :: Monad m => Float -> Stream m Word8

1350+ [A] float32be :: Monad m => Float -> Stream m Word8

1351+ [A] double64le :: Monad m => Double -> Stream m Word8

1352+ [A] double64be :: Monad m => Double -> Stream m Word8

1353+ [A] charUtf8 :: Monad m => Char -> Stream m Word8

1354+ [A] charLatin1 :: Applicative m => Char -> Stream m Word8

1355+ [A] bool :: Applicative m => Bool -> Stream m Word8

1356+[A] Streamly.Internal.Data.Binary.Parser

1357+ [A] class FromBytes a

1358+ [A] word8 :: Monad m => Parser Word8 m Word8

1359+ [A] word64le :: Monad m => Parser Word8 m Word64

1360+ [A] word64host :: MonadIO m => Parser Word8 m Word64

1361+ [A] word64be :: Monad m => Parser Word8 m Word64

1362+ [A] word32le :: Monad m => Parser Word8 m Word32

1363+ [A] word32be :: Monad m => Parser Word8 m Word32

1364+ [A] word16le :: Monad m => Parser Word8 m Word16

1365+ [A] word16be :: Monad m => Parser Word8 m Word16

1366+ [A] unit :: Monad m => Parser Word8 m ()

1367+ [A] ordering :: Monad m => Parser Word8 m Ordering

1368+ [A] int8 :: Monad m => Parser Word8 m Int8

1369+ [A] int64le :: Monad m => Parser Word8 m Int64

1370+ [A] int64be :: Monad m => Parser Word8 m Int64

1371+ [A] int32le :: Monad m => Parser Word8 m Int32

1372+ [A] int32be :: Monad m => Parser Word8 m Int32

1373+ [A] int16le :: Monad m => Parser Word8 m Int16

1374+ [A] int16be :: Monad m => Parser Word8 m Int16

1375+ [A] fromBytes :: FromBytes a => Parser Word8 m a

1376+ [A] float32le :: MonadIO m => Parser Word8 m Float

1377+ [A] float32be :: MonadIO m => Parser Word8 m Float

1378+ [A] eqWord8 :: Monad m => Word8 -> Parser Word8 m Word8

1379+ [A] double64le :: MonadIO m => Parser Word8 m Double

1380+ [A] double64be :: MonadIO m => Parser Word8 m Double

1381+ [A] charLatin1 :: Monad m => Parser Word8 m Char

1382+ [A] bool :: Monad m => Parser Word8 m Bool

1383+[R] Streamly.Internal.Data.Array.Type

1384+[A] Streamly.Internal.Data.Array.Stream

1385+ [A] unlines :: forall m a. (MonadIO m, Unbox a) => a -> Stream m (Array a) -> Stream m a

1386+ [A] toArray :: (MonadIO m, Unbox a) => Stream m (Array a) -> m (Array a)

1387+ [A] splitOnSuffix :: MonadIO m => Word8 -> Stream m (Array Word8) -> Stream m (Array Word8)

1388+ [A] splitOn :: MonadIO m => Word8 -> Stream m (Array Word8) -> Stream m (Array Word8)

1389+ [A] runArrayParserDBreak :: forall m a b. (MonadIO m, Unbox a) => Parser (Array a) m b -> Stream m (Array a) -> m (Either ParseError b, Stream m (Array a))

1390+ [A] runArrayFoldMany :: (Monad m, Unbox a) => ChunkFold m a b -> StreamK m (Array a) -> StreamK m (Either ParseError b)

1391+ [A] runArrayFoldBreak :: (MonadIO m, Unbox a) => ChunkFold m a b -> StreamK m (Array a) -> m (Either ParseError b, StreamK m (Array a))

1392+ [A] runArrayFold :: (MonadIO m, Unbox a) => ChunkFold m a b -> StreamK m (Array a) -> m (Either ParseError b)

1393+ [A] pinnedChunksOf :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> Stream m (Array a)

1394+ [A] parseChunks :: (Monad m, Unbox a) => ParserK (Array a) m b -> StreamK m (Array a) -> m (Either ParseError b)

1395+ [A] parseBreakChunks :: (Monad m, Unbox a) => ParserK (Array a) m b -> StreamK m (Array a) -> m (Either ParseError b, StreamK m (Array a))

1396+ [A] parseBreak :: (MonadIO m, Unbox a) => Parser a m b -> StreamK m (Array a) -> m (Either ParseError b, StreamK m (Array a))

1397+ [A] lpackArraysChunksOf :: (MonadIO m, Unbox a) => Int -> Fold m (Array a) () -> Fold m (Array a) ()

1398+ [A] interposeSuffix :: (Monad m, Unbox a) => a -> Stream m (Array a) -> Stream m a

1399+ [A] interpose :: (Monad m, Unbox a) => a -> Stream m (Array a) -> Stream m a

1400+ [A] intercalateSuffix :: (Monad m, Unbox a) => Array a -> Stream m (Array a) -> Stream m a

1401+ [A] foldBreakD :: forall m a b. (MonadIO m, Unbox a) => Fold m a b -> Stream m (Array a) -> m (b, Stream m (Array a))

1402+ [A] foldBreak :: (MonadIO m, Unbox a) => Fold m a b -> StreamK m (Array a) -> m (b, StreamK m (Array a))

1403+ [A] flattenArraysRev :: forall m a. (MonadIO m, Unbox a) => Stream m (Array a) -> Stream m a

1404+ [A] flattenArrays :: forall m a. (MonadIO m, Unbox a) => Stream m (Array a) -> Stream m a

1405+ [A] concatRev :: (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

1406+ [A] concat :: (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

1407+ [A] compact :: (MonadIO m, Unbox a) => Int -> Stream m (Array a) -> Stream m (Array a)

1408+ [A] chunksOf :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> Stream m (Array a)

1409+ [A] bufferChunks :: (MonadIO m, Unbox a) => Stream m a -> m (StreamK m (Array a))

1410+[R] Streamly.Internal.Data.Array.Mut.Type

1411+[R] Streamly.Internal.Data.Array.Mut.Stream

1412+[R] Streamly.Internal.Data.Array.Mut

1413+[R] Streamly.Internal.Data.Array.Generic.Mut.Type

1414+[C] Streamly.Internal.Data.Array.Generic

1415+ [A] getIndex :: Int -> Array a -> Maybe a

1416+ [A] fromPureStream :: Stream Identity a -> Array a

1417+ [A] fromByteStr# :: Addr# -> Array Word8

1418+ [A] chunksOf :: forall m a. MonadIO m => Int -> Stream m a -> Stream m (Array a)

1419+[C] Streamly.Internal.Data.Array

1420+ [A] ArrayUnsafe

1421+ [A] ArrayUnsafe :: {-# UNPACK #-} !MutByteArray -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> ArrayUnsafe a

1422+ [C] Array

1423+ [A] [arrStart] :: Array a -> {-# UNPACK #-} !Int

1424+ [A] [arrEnd] :: Array a -> {-# UNPACK #-} !Int

1425+ [A] [arrContents] :: Array a -> {-# UNPACK #-} !MutByteArray

1426+ [A] Array :: {-# UNPACK #-} !MutByteArray -> {-# UNPACK #-} !Int -> {-# UNPACK #-} !Int -> Array a

1427+ [A] writeWith :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

1428+ [A] writeNUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

1429+ [R] writeNAligned :: forall m a. (MonadIO m, Unbox a) => Int -> Int -> Fold m a (Array a)

1430+ [A] unsafeMakePure :: Monad m => Fold IO a b -> Fold m a b

1431+ [A] unsafeIndexIO :: forall a. Unbox a => Int -> Array a -> IO a

1432+ [D] unsafeIndex :: forall a. Unbox a => Int -> Array a -> a

1433+ [A] unsafeFreezeWithShrink :: Unbox a => MutArray a -> Array a

1434+ [A] unpin :: Array a -> IO (Array a)

1435+ [A] toStreamKRev :: forall m a. (Monad m, Unbox a) => Array a -> StreamK m a

1436+ [A] toStreamK :: forall m a. (Monad m, Unbox a) => Array a -> StreamK m a

1437+ [A] toStreamDRev :: forall m a. (Monad m, Unbox a) => Array a -> Stream m a

1438+ [A] toStreamD :: forall m a. (Monad m, Unbox a) => Array a -> Stream m a

1439+ [A] splitAt :: Unbox a => Int -> Array a -> (Array a, Array a)

1440+ [A] splice :: (MonadIO m, Unbox a) => Array a -> Array a -> m (Array a)

1441+ [A] serialize :: Serialize a => a -> Array Word8

1442+ [A] pinnedWriteNUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

1443+ [A] pinnedWriteNAligned :: forall m a. (MonadIO m, Unbox a) => Int -> Int -> Fold m a (Array a)

1444+ [A] pinnedWriteN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

1445+ [A] pinnedWrite :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

1446+ [A] pinnedSerialize :: Serialize a => a -> Array Word8

1447+ [A] pinnedFromListN :: Unbox a => Int -> [a] -> Array a

1448+ [A] pinnedFromList :: Unbox a => [a] -> Array a

1449+ [A] pinnedClone :: MonadIO m => Array a -> m (Array a)

1450+ [A] pinnedChunksOf :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> Stream m (Array a)

1451+ [A] pin :: Array a -> IO (Array a)

1452+ [A] nil :: Array a

1453+ [A] isPinned :: Array a -> Bool

1454+ [A] getIndexUnsafe :: forall a. Unbox a => Int -> Array a -> a

1455+ [A] fromStreamDN :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> m (Array a)

1456+ [A] fromStreamD :: forall m a. (MonadIO m, Unbox a) => Stream m a -> m (Array a)

1457+ [A] fromPureStream :: Unbox a => Stream Identity a -> Array a

1458+ [A] fromListRevN :: Unbox a => Int -> [a] -> Array a

1459+ [A] fromListRev :: Unbox a => [a] -> Array a

1460+ [A] fromByteStr# :: Addr# -> Array Word8

1461+ [A] foldr :: Unbox a => (a -> b -> b) -> b -> Array a -> b

1462+ [A] foldl' :: forall a b. Unbox a => (b -> a -> b) -> b -> Array a -> b

1463+ [A] flattenArraysRev :: forall m a. (MonadIO m, Unbox a) => Stream m (Array a) -> Stream m a

1464+ [A] flattenArrays :: forall m a. (MonadIO m, Unbox a) => Stream m (Array a) -> Stream m a

1465+ [A] encodeAs :: forall a. Serialize a => PinnedState -> a -> Array Word8

1466+ [A] deserialize :: Serialize a => Array Word8 -> a

1467+ [A] clone :: MonadIO m => Array a -> m (Array a)

1468+ [A] chunksOf :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> Stream m (Array a)

1469+ [A] byteLength :: Array a -> Int

1470+ [A] bufferChunks :: (MonadIO m, Unbox a) => Stream m a -> m (StreamK m (Array a))

1471+ [A] breakOn :: MonadIO m => Word8 -> Array Word8 -> m (Array Word8, Maybe (Array Word8))

core/docs/ApiChangelogs/0.2.0-0.2.2.txtnew file+306

@@ -0,0 +1,306 @@

1+---------------------------------

2+API Annotations

3+---------------------------------

5+[A] : Added

6+[R] : Removed

7+[C] : Changed

8+[O] : Old definition

9+[N] : New definition

10+[D] : Deprecated

11+

12+---------------------------------

13+API diff

14+---------------------------------

15+

16+[C] Streamly.Data.Stream

17+ [A] (FixityR,5)

18+ [A] (FixityR,5)

19+[C] Streamly.Data.MutArray.Generic

20+ [A] emptyOf :: MonadIO m => Int -> m (MutArray a)

21+ [A] createOf :: MonadIO m => Int -> Fold m a (MutArray a)

22+ [A] create :: MonadIO m => Fold m a (MutArray a)

23+[C] Streamly.Data.MutArray

24+ [A] pinnedEmptyOf :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a)

25+ [A] emptyOf :: (MonadIO m, Unbox a) => Int -> m (MutArray a)

26+ [A] createOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

27+ [A] create :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)

28+ [A] appendN :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a) -> Fold m a (MutArray a)

29+ [A] append :: forall m a. (MonadIO m, Unbox a) => m (MutArray a) -> Fold m a (MutArray a)

30+[C] Streamly.Data.Array.Generic

31+ [A] createOf :: MonadIO m => Int -> Fold m a (Array a)

32+ [A] create :: MonadIO m => Fold m a (Array a)

33+[C] Streamly.Data.Array

34+ [A] class Serialize a

35+ [A] serializeAt :: Serialize a => Int -> MutByteArray -> a -> IO Int

36+ [A] pinnedSerialize :: Serialize a => a -> Array Word8

37+ [A] deserializeAt :: Serialize a => Int -> MutByteArray -> Int -> IO (Int, a)

38+ [A] deserialize :: Serialize a => Array Word8 -> a

39+ [A] createOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

40+ [A] create :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

41+ [A] addSizeTo :: Serialize a => Int -> a -> Int

42+

43+---------------------------------

44+Internal API diff

45+---------------------------------

46+

47+[C] Streamly.Internal.Unicode.Stream

48+ [A] encodeUtf16le' :: Stream m Char -> Stream m Word16

49+ [A] decodeUtf16le' :: Stream m Word16 -> Stream m Char

50+[A] Streamly.Internal.FileSystem.Path

51+ [A] class IsPath a

52+ [A] Rel

53+ [A] File

54+ [A] Dir

55+ [A] Abs

56+ [A] Streamly.Internal.FileSystem.Path.IsPath

57+ [A] instance Streamly.Internal.FileSystem.Path.IsPath Streamly.Internal.FileSystem.Path.Path

58+ [A] instance Streamly.Internal.FileSystem.Path.IsPath (Streamly.Internal.FileSystem.Path.Rel Streamly.Internal.FileSystem.Path.Path)

59+ [A] instance Streamly.Internal.FileSystem.Path.IsPath (Streamly.Internal.FileSystem.Path.Rel (Streamly.Internal.FileSystem.Path.File Streamly.Internal.FileSystem.Path.Path))

60+ [A] instance Streamly.Internal.FileSystem.Path.IsPath (Streamly.Internal.FileSystem.Path.Rel (Streamly.Internal.FileSystem.Path.Dir Streamly.Internal.FileSystem.Path.Path))

61+ [A] instance Streamly.Internal.FileSystem.Path.IsPath (Streamly.Internal.FileSystem.Path.File Streamly.Internal.FileSystem.Path.Path)

62+ [A] instance Streamly.Internal.FileSystem.Path.IsPath (Streamly.Internal.FileSystem.Path.Dir Streamly.Internal.FileSystem.Path.Path)

63+ [A] instance Streamly.Internal.FileSystem.Path.IsPath (Streamly.Internal.FileSystem.Path.Abs Streamly.Internal.FileSystem.Path.Path)

64+ [A] instance Streamly.Internal.FileSystem.Path.IsPath (Streamly.Internal.FileSystem.Path.Abs (Streamly.Internal.FileSystem.Path.File Streamly.Internal.FileSystem.Path.Path))

65+ [A] instance Streamly.Internal.FileSystem.Path.IsPath (Streamly.Internal.FileSystem.Path.Abs (Streamly.Internal.FileSystem.Path.Dir Streamly.Internal.FileSystem.Path.Path))

66+ [A] GHC.Show.Show

67+ [A] instance GHC.Show.Show Streamly.Internal.FileSystem.Path.PathException

68+ [A] GHC.Exception.Type.Exception

69+ [A] instance GHC.Exception.Type.Exception Streamly.Internal.FileSystem.Path.PathException

70+ [A] GHC.Classes.Eq

71+ [A] instance GHC.Classes.Eq Streamly.Internal.FileSystem.Path.PathException

72+ [A] Path

73+ [A] Path :: Array Word8 -> Path

74+ [A] toString :: Path -> [Char]

75+ [A] toPath :: IsPath a => a -> Path

76+ [A] toChunk :: Path -> Array Word8

77+ [A] toChars :: Monad m => Path -> Stream m Char

78+ [A] relfile :: QuasiQuoter

79+ [A] reldir :: QuasiQuoter

80+ [A] rel :: QuasiQuoter

81+ [A] primarySeparator :: Char

82+ [A] path :: QuasiQuoter

83+ [A] mkRelFile :: String -> Q Exp

84+ [A] mkRelDir :: String -> Q Exp

85+ [A] mkRel :: String -> Q Exp

86+ [A] mkPath :: String -> Q Exp

87+ [A] mkFile :: String -> Q Exp

88+ [A] mkDir :: String -> Q Exp

89+ [A] mkAbsFile :: String -> Q Exp

90+ [A] mkAbsDir :: String -> Q Exp

91+ [A] mkAbs :: String -> Q Exp

92+ [A] isSeparator :: Char -> Bool

93+ [A] fromString :: MonadThrow m => [Char] -> m Path

94+ [A] fromPathUnsafe :: IsPath a => Path -> a

95+ [A] fromPath :: (IsPath a, MonadThrow m) => Path -> m a

96+ [A] fromChunkUnsafe :: Array Word8 -> Path

97+ [A] fromChunk :: MonadThrow m => Array Word8 -> m Path

98+ [A] fromChars :: MonadThrow m => Stream Identity Char -> m Path

99+ [A] file :: QuasiQuoter

100+ [A] extendPath :: Path -> Path -> Path

101+ [A] extendDir :: (IsPath (a (Dir Path)), IsPath b, IsPath (a b)) => a (Dir Path) -> Rel b -> a b

102+ [A] dir :: QuasiQuoter

103+ [A] adaptPath :: (MonadThrow m, IsPath a, IsPath b) => a -> m b

104+ [A] absfile :: QuasiQuoter

105+ [A] absdir :: QuasiQuoter

106+ [A] abs :: QuasiQuoter

107+[C] Streamly.Internal.Data.Stream

108+ [A] (FixityR,5)

109+ [A] (FixityR,5)

110+ [C] splitInnerBySuffix

111+ [O] splitInnerBySuffix :: (Monad m, Eq (f a), Monoid (f a)) => (f a -> m (f a, Maybe (f a))) -> (f a -> f a -> m (f a)) -> Stream m (f a) -> Stream m (f a)

112+ [N] splitInnerBySuffix :: Monad m => (f a -> Bool) -> (f a -> m (f a, Maybe (f a))) -> (f a -> f a -> m (f a)) -> Stream m (f a) -> Stream m (f a)

113+ [D] sliceOnSuffix :: Monad m => (a -> Bool) -> Stream m a -> Stream m (Int, Int)

114+ [A] indexOnSuffix :: Monad m => (a -> Bool) -> Stream m a -> Stream m (Int, Int)

115+[C] Streamly.Internal.Data.MutByteArray

116+ [A] unsafePinnedAsPtr :: MonadIO m => MutByteArray -> (Ptr a -> m b) -> m b

117+ [A] unsafeAsPtr :: MonadIO m => MutByteArray -> (Ptr a -> m b) -> m b

118+ [D] nil :: MutByteArray

119+ [A] empty :: MutByteArray

120+ [D] asPtrUnsafe :: MonadIO m => MutByteArray -> (Ptr a -> m b) -> m b

121+[D] Streamly.Internal.Data.MutArray.Stream

122+ [D] writeChunks :: (MonadIO m, Unbox a) => Int -> Fold m a (StreamK n (MutArray a))

123+ [D] fromArrayStreamK :: (Unbox a, MonadIO m) => StreamK m (MutArray a) -> m (MutArray a)

124+ [D] flattenArraysRev :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> Stream m a

125+ [D] flattenArrays :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> Stream m a

126+[C] Streamly.Internal.Data.MutArray.Generic

127+ [D] writeWith :: MonadIO m => Int -> Fold m a (MutArray a)

128+ [D] writeNUnsafe :: MonadIO m => Int -> Fold m a (MutArray a)

129+ [A] unsafeCreateOf :: MonadIO m => Int -> Fold m a (MutArray a)

130+ [A] emptyOf :: MonadIO m => Int -> m (MutArray a)

131+ [A] createWith :: MonadIO m => Int -> Fold m a (MutArray a)

132+ [A] createOf :: MonadIO m => Int -> Fold m a (MutArray a)

133+ [A] create :: MonadIO m => Fold m a (MutArray a)

134+[C] Streamly.Internal.Data.MutArray

135+ [A] SpliceState

136+ [A] SpliceYielding :: arr -> SpliceState s arr -> SpliceState s arr

137+ [A] SpliceInitial :: s -> SpliceState s arr

138+ [A] SpliceFinish :: SpliceState s arr

139+ [A] SpliceBuffering :: s -> arr -> SpliceState s arr

140+ [R] MutByteArray

141+ [D] writeWith :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

142+ [D] writeRevN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

143+ [D] writeNWithUnsafe :: forall m a. (MonadIO m, Unbox a) => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)

144+ [D] writeNUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

145+ [D] writeChunks :: (MonadIO m, Unbox a) => Int -> Fold m a (StreamK n (MutArray a))

146+ [D] writeAppendWith :: forall m a. (MonadIO m, Unbox a) => (Int -> Int) -> m (MutArray a) -> Fold m a (MutArray a)

147+ [D] writeAppendNUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a) -> Fold m a (MutArray a)

148+ [A] unsafePinnedCreateOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

149+ [A] unsafePinnedAsPtr :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b

150+ [A] unsafeCreateOfWith :: forall m a. (MonadIO m, Unbox a) => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)

151+ [A] unsafeCreateOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

152+ [A] unsafeAsPtr :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b

153+ [A] unsafeAppendN :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a) -> Fold m a (MutArray a)

154+ [A] toStreamWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> MutArray a -> Stream m a

155+ [A] toStreamRevWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> MutArray a -> Stream m a

156+ [R] toStreamDWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> MutArray a -> Stream m a

157+ [R] toStreamDRevWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> MutArray a -> Stream m a

158+ [A] slicerFromLen :: forall m a. (Monad m, Unbox a) => Int -> Int -> Unfold m (MutArray a) (MutArray a)

159+ [A] sliceIndexerFromLen :: forall m a. (Monad m, Unbox a) => Int -> Int -> Unfold m (MutArray a) (Int, Int)

160+ [A] revCreateOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

161+ [D] resizeExp :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)

162+ [D] resize :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)

163+ [A] pokeSkipUnsafe :: Int -> MutArray Word8 -> MutArray Word8

164+ [A] pokeAppendMay :: forall m a. (MonadIO m, Unbox a) => MutArray Word8 -> a -> m (Maybe (MutArray Word8))

165+ [A] pokeAppend :: forall m a. (MonadIO m, Unbox a) => MutArray Word8 -> a -> m (MutArray Word8)

166+ [D] pinnedWriteNUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

167+ [D] pinnedWriteN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

168+ [D] pinnedWrite :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)

169+ [D] pinnedNewBytes :: MonadIO m => Int -> m (MutArray a)

170+ [A] pinnedEmptyOf :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a)

171+ [A] pinnedCreateOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

172+ [A] pinnedCreate :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)

173+ [A] pinnedCompactLE :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

174+ [A] peekUnconsUnsafe :: forall m a. (MonadIO m, Unbox a) => MutArray Word8 -> m (a, MutArray Word8)

175+ [A] peekUncons :: forall m a. (MonadIO m, Unbox a) => MutArray Word8 -> m (Maybe a, MutArray Word8)

176+ [A] peekSkipUnsafe :: Int -> MutArray Word8 -> MutArray Word8

177+ [A] pPinnedCompactLE :: forall m a. (MonadIO m, Unbox a) => Int -> Parser (MutArray a) m (MutArray a)

178+ [A] pCompactLE :: forall m a. (MonadIO m, Unbox a) => Int -> Parser (MutArray a) m (MutArray a)

179+ [D] nil :: MutArray a

180+ [A] lPinnedCompactGE :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()

181+ [A] lCompactGE :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()

182+ [A] indexReaderWith :: (Monad m, Unbox a) => (forall b. IO b -> m b) -> Stream m Int -> Unfold m (MutArray a) a

183+ [A] indexReader :: (MonadIO m, Unbox a) => Stream m Int -> Unfold m (MutArray a) a

184+ [A] growExp :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)

185+ [A] grow :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)

186+ [D] getSlicesFromLen :: forall m a. (Monad m, Unbox a) => Int -> Int -> Unfold m (MutArray a) (MutArray a)

187+ [R] getIndicesD :: (Monad m, Unbox a) => (forall b. IO b -> m b) -> Stream m Int -> Unfold m (MutArray a) a

188+ [D] getIndices :: (MonadIO m, Unbox a) => Stream m Int -> Unfold m (MutArray a) a

189+ [D] genSlicesFromLen :: forall m a. (Monad m, Unbox a) => Int -> Int -> Unfold m (MutArray a) (Int, Int)

190+ [A] fromStreamN :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> m (MutArray a)

191+ [D] fromStreamDN :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> m (MutArray a)

192+ [D] fromStreamD :: (MonadIO m, Unbox a) => Stream m a -> m (MutArray a)

193+ [A] fromPureStreamN :: (MonadIO m, Unbox a) => Int -> Stream Identity a -> m (MutArray a)

194+ [A] fromPtrN :: MonadIO m => Int -> Ptr Word8 -> m (MutArray Word8)

195+ [A] fromChunksRealloced :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> m (MutArray a)

196+ [A] fromChunksK :: (Unbox a, MonadIO m) => StreamK m (MutArray a) -> m (MutArray a)

197+ [A] fromByteStr# :: MonadIO m => Addr# -> m (MutArray Word8)

198+ [D] fromArrayStreamK :: (Unbox a, MonadIO m) => StreamK m (MutArray a) -> m (MutArray a)

199+ [D] flattenArraysRev :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> Stream m a

200+ [D] flattenArrays :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> Stream m a

201+ [A] fPinnedCompactGE :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m (MutArray a) (MutArray a)

202+ [A] fCompactGE :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m (MutArray a) (MutArray a)

203+ [A] emptyOf :: (MonadIO m, Unbox a) => Int -> m (MutArray a)

204+ [A] empty :: MutArray a

205+ [A] createWith :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

206+ [A] createOfWith :: forall m a. (MonadIO m, Unbox a) => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)

207+ [A] createOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

208+ [A] create :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)

209+ [A] concatWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> Stream m (MutArray a) -> Stream m a

210+ [A] concatRevWith :: forall m a. (Monad m, Unbox a) => (forall b. IO b -> m b) -> Stream m (MutArray a) -> Stream m a

211+ [A] concatRev :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> Stream m a

212+ [A] concat :: forall m a. (MonadIO m, Unbox a) => Stream m (MutArray a) -> Stream m a

213+ [A] compactOnByteSuffix :: MonadIO m => Word8 -> Stream m (MutArray Word8) -> Stream m (MutArray Word8)

214+ [A] compactOnByte :: MonadIO m => Word8 -> Stream m (MutArray Word8) -> Stream m (MutArray Word8)

215+ [A] compactLeAs :: forall m a. (MonadIO m, Unbox a) => PinnedState -> Int -> Stream m (MutArray a) -> Stream m (MutArray a)

216+ [A] compactLE :: (MonadIO m, Unbox a) => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

217+ [A] compactGE :: (MonadIO m, Unbox a) => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

218+ [A] compactEQ :: Int -> Stream m (MutArray a) -> Stream m (MutArray a)

219+ [D] cmp :: MonadIO m => MutArray a -> MutArray a -> m Ordering

220+ [A] byteEq :: MonadIO m => MutArray a -> MutArray a -> m Bool

221+ [A] byteCmp :: MonadIO m => MutArray a -> MutArray a -> m Ordering

222+ [A] buildChunks :: (MonadIO m, Unbox a) => Int -> Fold m a (StreamK n (MutArray a))

223+ [D] asPtrUnsafe :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b

224+ [A] appendWith :: forall m a. (MonadIO m, Unbox a) => (Int -> Int) -> m (MutArray a) -> Fold m a (MutArray a)

225+ [A] appendN :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a) -> Fold m a (MutArray a)

226+ [A] append :: forall m a. (MonadIO m, Unbox a) => m (MutArray a) -> Fold m a (MutArray a)

227+[D] Streamly.Internal.Data.Array.Stream

228+ [C] interposeSuffix

229+ [O] interposeSuffix :: (Monad m, Unbox a) => a -> Stream m (Array a) -> Stream m a

230+ [N] interposeSuffix :: forall m a. (Monad m, Unbox a) => a -> Stream m (Array a) -> Stream m a

231+ [D] flattenArraysRev :: forall m a. (MonadIO m, Unbox a) => Stream m (Array a) -> Stream m a

232+ [D] flattenArrays :: forall m a. (MonadIO m, Unbox a) => Stream m (Array a) -> Stream m a

233+ [C] concatRev

234+ [O] concatRev :: (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

235+ [N] concatRev :: forall m a. (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

236+ [D] bufferChunks :: (MonadIO m, Unbox a) => Stream m a -> m (StreamK m (Array a))

237+[C] Streamly.Internal.Data.Array.Generic

238+ [A] createOf :: MonadIO m => Int -> Fold m a (Array a)

239+ [A] create :: MonadIO m => Fold m a (Array a)

240+[C] Streamly.Internal.Data.Array

241+ [R] ArrayUnsafe

242+ [D] writeWith :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

243+ [D] writeNUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

244+ [A] unsafePinnedCreateOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

245+ [A] unsafePinnedAsPtr :: MonadIO m => Array a -> (Ptr a -> m b) -> m b

246+ [A] unsafeCreateOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

247+ [D] toStreamDRev :: forall m a. (Monad m, Unbox a) => Array a -> Stream m a

248+ [D] toStreamD :: forall m a. (Monad m, Unbox a) => Array a -> Stream m a

249+ [C] splice

250+ [O] splice :: (MonadIO m, Unbox a) => Array a -> Array a -> m (Array a)

251+ [N] splice :: MonadIO m => Array a -> Array a -> m (Array a)

252+ [A] slicerFromLen :: forall m a. (Monad m, Unbox a) => Int -> Int -> Unfold m (Array a) (Array a)

253+ [A] sliceIndexerFromLen :: forall m a. (Monad m, Unbox a) => Int -> Int -> Unfold m (Array a) (Int, Int)

254+ [D] pinnedWriteNUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

255+ [D] pinnedWriteNAligned :: forall m a. (MonadIO m, Unbox a) => Int -> Int -> Fold m a (Array a)

256+ [D] pinnedWriteN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

257+ [D] pinnedWrite :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

258+ [A] pinnedCreateOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

259+ [A] pinnedCreate :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

260+ [A] pinnedCompactLE :: (MonadIO m, Unbox a) => Int -> Stream m (Array a) -> Stream m (Array a)

261+ [A] parseBreakChunksK :: forall m a b. (MonadIO m, Unbox a) => Parser a m b -> StreamK m (Array a) -> m (Either ParseError b, StreamK m (Array a))

262+ [D] nil :: Array a

263+ [A] lPinnedCompactGE :: (MonadIO m, Unbox a) => Int -> Fold m (Array a) () -> Fold m (Array a) ()

264+ [A] lCompactGE :: (MonadIO m, Unbox a) => Int -> Fold m (Array a) () -> Fold m (Array a) ()

265+ [A] interposeSuffix :: forall m a. (Monad m, Unbox a) => a -> Stream m (Array a) -> Stream m a

266+ [A] interpose :: (Monad m, Unbox a) => a -> Stream m (Array a) -> Stream m a

267+ [A] intercalateSuffix :: (Monad m, Unbox a) => Array a -> Stream m (Array a) -> Stream m a

268+ [A] indexReaderFromThenTo :: Unfold m (Int, Int, Int, Array a) a

269+ [A] indexReader :: (Monad m, Unbox a) => Stream m Int -> Unfold m (Array a) a

270+ [A] indexFinder :: (a -> Bool) -> Unfold Identity (Array a) Int

271+ [D] getSlicesFromLen :: forall m a. (Monad m, Unbox a) => Int -> Int -> Unfold m (Array a) (Array a)

272+ [R] getIndicesFromThenTo :: Unfold m (Int, Int, Int, Array a) a

273+ [D] getIndices :: (Monad m, Unbox a) => Stream m Int -> Unfold m (Array a) a

274+ [D] genSlicesFromLen :: forall m a. (Monad m, Unbox a) => Int -> Int -> Unfold m (Array a) (Int, Int)

275+ [D] fromStreamDN :: forall m a. (MonadIO m, Unbox a) => Int -> Stream m a -> m (Array a)

276+ [D] fromStreamD :: forall m a. (MonadIO m, Unbox a) => Stream m a -> m (Array a)

277+ [A] fromPureStreamN :: Unbox a => Int -> Stream Identity a -> Array a

278+ [A] fromPtrN :: Int -> Ptr Word8 -> Array Word8

279+ [A] fromChunksK :: (MonadIO m, Unbox a) => StreamK m (Array a) -> m (Array a)

280+ [A] fromChunks :: (MonadIO m, Unbox a) => Stream m (Array a) -> m (Array a)

281+ [A] fromByteStr :: Ptr Word8 -> Array Word8

282+ [A] foldChunks :: (MonadIO m, Unbox a) => Fold m a b -> Stream m (Array a) -> m b

283+ [A] foldBreakChunksK :: forall m a b. (MonadIO m, Unbox a) => Fold m a b -> StreamK m (Array a) -> m (b, StreamK m (Array a))

284+ [A] foldBreakChunks :: forall m a b. (MonadIO m, Unbox a) => Fold m a b -> Stream m (Array a) -> m (b, Stream m (Array a))

285+ [D] flattenArraysRev :: forall m a. (MonadIO m, Unbox a) => Stream m (Array a) -> Stream m a

286+ [D] flattenArrays :: forall m a. (MonadIO m, Unbox a) => Stream m (Array a) -> Stream m a

287+ [C] findIndicesOf

288+ [O] findIndicesOf :: (a -> Bool) -> Unfold Identity (Array a) Int

289+ [N] findIndicesOf :: (a -> Bool) -> Array a -> Stream Identity Int

290+ [A] fPinnedCompactGE :: (MonadIO m, Unbox a) => Int -> Fold m (Array a) (Array a)

291+ [A] fCompactGE :: (MonadIO m, Unbox a) => Int -> Fold m (Array a) (Array a)

292+ [A] empty :: Array a

293+ [A] createWith :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

294+ [A] createOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

295+ [A] create :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

296+ [A] concatRev :: forall m a. (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

297+ [A] concat :: (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

298+ [A] compactOnByteSuffix :: MonadIO m => Word8 -> Stream m (Array Word8) -> Stream m (Array Word8)

299+ [A] compactOnByte :: MonadIO m => Word8 -> Stream m (Array Word8) -> Stream m (Array Word8)

300+ [A] compactLE :: (MonadIO m, Unbox a) => Int -> Stream m (Array a) -> Stream m (Array a)

301+ [A] compactGE :: (MonadIO m, Unbox a) => Int -> Stream m (Array a) -> Stream m (Array a)

302+ [A] byteEq :: Array a -> Array a -> Bool

303+ [A] byteCmp :: Array a -> Array a -> Ordering

304+ [A] buildChunks :: (MonadIO m, Unbox a) => Stream m a -> m (StreamK m (Array a))

305+ [D] bufferChunks :: (MonadIO m, Unbox a) => Stream m a -> m (StreamK m (Array a))

306+ [D] asPtrUnsafe :: MonadIO m => Array a -> (Ptr a -> m b) -> m b

core/docs/Changelog.md+58−16

@@ -1,31 +1,73 @@

11# Changelog

3−## Unreleased

3+## 0.2.2 (Jan 2024)

5−* Arrays are now created unpinned by default, they were created pinned

6− earlier. During IO operations unpinned arrays are automatically copied

7− to pinned memory. When arrays are directly passed to IO operations

8− programmers can choose to create them pinned to avoid a copy. To

9− create pinned arrays, use the internal APIs with the `pinned*` prefix.

5+* Add fixities `infixr 5` for `cons` and `consM` functions.

6+* Fix a bug in Array `Eq` instance when the type is a sum type with

7+ differently sized constructors.

8+* lpackArraysChunksOf, compact, writeChunksWith, putChunksWith now take the

9+ buffer size in number of array elements instead of bytes.

1010

11−* Changed the signature of 'Streamly.Data.Stream.handle' to make the exception

12− handler monadic.

13−* Rethrow the exception promptly in bracketIO.

11+## 0.2.1 (Dec 2023)

1412

15−### Breaking

13+* Make the serialization of the unit constructor deterministic.

14+* Expose `pinnedSerialize` & `deserialize` via `Streamly.Data.Array`.

1615

17−* `ParserK` in `Streamly.Data.ParserK` is not specialized to arrays anymore. To

18− adapt to the new code, change `ParserK a m b` to `ParserK (Array a) m b` where

19− `Array` comes from `Streamly.Data.Array`.

16+## 0.2.0 (Nov 2023)

17+

18+See [0.1.0-0.2.0 API Changelog](https://github.com/composewell/streamly/blob/streamly-0.10.0/core/docs/ApiChangelogs/0.1.0-0.2.0.txt)

19+for a full list of API changes in this release. Only a few significant

20+changes are mentioned here.

21+

22+### Breaking Changes

23+

24+* `ParserK` in `Streamly.Data.ParserK` is not implicitly specialized

25+ to arrays anymore. To adapt to the new code, change `ParserK a m

26+ b` to `ParserK (Array a) m b` where the `Array` type comes from

27+ `Streamly.Data.Array`. This change also affected the signatures of

28+ `parseChunks` and `parseBreakChunks`.

29+* Changed the signature of 'Streamly.Data.Stream.handle' to make the

30+ exception handler monadic.

31+* Behavior change: Exceptions are now rethrown promptly in `bracketIO`.

32+

33+### Enhancements

34+

35+* __Serialization__: Added a `Streamly.Data.MutByteArray` module with a

36+ `Serialize` type class for fast binary serialization. The Data.Array

37+ module supplies the `serialize` and `deserialize` operations for arrays.

38+* __Unpinned Arrays__: Unboxed arrays are now created unpinned by default,

39+ they were created pinned earlier. During IO operations, unpinned arrays

40+ are automatically copied to pinned memory. When arrays are directly

41+ passed to IO operations programmers can choose to create them pinned to

42+ avoid a copy. To create pinned arrays, use the internal APIs with the

43+ `pinned*` prefix.

44+* StreamK now supports native exception handling routines (handle, bracketIO).

45+ Earlier we had to convert it to the `Stream` type for exception handling.

2046

2147### Deprecations

2248

23−* `Streamly.Data.MutArray.newUnpinned` is renamed to

24− `Streamly.Data.MutArray.pinnedNew`

49+See [0.1.0-0.2.0 API Changelog](https://github.com/composewell/streamly/blob/streamly-0.10.0/core/docs/ApiChangelogs/0.1.0-0.2.0.txt)

50+for a full list of deprecations.

51+

52+### Internal API Changes

53+

54+* Fold constructor has changed, added a `final` field to support

55+ finalization and cleanup of a chain of folds. The `extract` field is

56+ now used only for mapping the fold internal state to fold result for

57+ scanning purposes. If your fold does not require cleanup you can just use

58+ your existing `extract` function as `final` as well to adapt to this change.

59+* Many low level internal modules have been removed, they are entirely

60+ exported from higher level internal modules. If you were importing any

61+ of the missing low level modules then import the higher level modules instead.

62+* Internal module changes:

63+ * Streamly.Internal.Serialize.FromBytes -> Streamly.Internal.Data.Binary.Parser

64+ * Streamly.Internal.Serialize.ToBytes -> Streamly.Internal.Data.Binary.Stream

65+ * Streamly.Internal.Data.Unbox is now exported via Streamly.Internal.Data.Serialize

66+ * Streamly.Internal.Data.IORef.Unboxed is now exported via Streamly.Internal.Data.Serialize

2567

2668## 0.1.0 (March 2023)

2769

28−Also see [streamly-core-0.1.0 API Changelog](/core/docs/ApiChangelogs/0.1.0.txt) or

70+Also see [streamly-core-0.1.0 API Changelog](https://github.com/composewell/streamly/blob/streamly-0.10.0/core/docs/ApiChangelogs/0.1.0.txt) or

2971https://hackage.haskell.org/package/streamly-core-0.1.0/docs/docs/ApiChangelogs/0.1.0.txt

3072

3173`streamly` package is split into two packages, (1) `streamly-core` that

core/src/DocTestDataArray.hs+3

@@ -11,9 +11,12 @@

1111

1212>>> import qualified Streamly.Data.Array as Array

1313>>> import qualified Streamly.Data.Fold as Fold

14+>>> import qualified Streamly.Data.ParserK as ParserK

1415>>> import qualified Streamly.Data.Stream as Stream

16+>>> import qualified Streamly.Data.StreamK as StreamK

1517

1618For APIs that have not been released yet.

1719

1820>>> import qualified Streamly.Internal.Data.Array as Array

21+>>> import qualified Streamly.Internal.Data.Stream as Stream

1922-}

core/src/DocTestDataParserK.hsnew file+15

@@ -0,0 +1,15 @@

1+{- $setup

2+>>> :m

3+>>> import Control.Applicative ((<|>))

4+>>> import Data.Char (isDigit, isAlpha)

6+>>> import Streamly.Data.Parser (Parser)

7+>>> import Streamly.Data.ParserK (ParserK)

9+>>> import qualified Streamly.Data.Parser as Parser

10+>>> import qualified Streamly.Data.ParserK as ParserK

11+

12+For APIs that have not been released yet.

13+

14+>>> import qualified Streamly.Internal.Data.ParserK as ParserK

15+-}

core/src/DocTestDataStream.hs+2

@@ -6,6 +6,7 @@

66>>> import Control.Monad.IO.Class (MonadIO (liftIO))

77>>> import Control.Monad.Trans.Class (lift)

88>>> import Control.Monad.Trans.Identity (runIdentityT)

9+>>> import Data.Char (isSpace)

910>>> import Data.Either (fromLeft, fromRight, isLeft, isRight, either)

1011>>> import Data.Maybe (fromJust, isJust)

1112>>> import Data.Function (fix, (&))

@@ -32,6 +33,7 @@ For APIs that have not been released yet.

3233>>> import qualified Streamly.Internal.Data.Fold as Fold

3334>>> import qualified Streamly.Internal.Data.Parser as Parser

3435>>> import qualified Streamly.Internal.Data.Stream as Stream

36+>>> import qualified Streamly.Internal.Data.StreamK as StreamK

3537>>> import qualified Streamly.Internal.Data.Unfold as Unfold

3638>>> import qualified Streamly.Internal.FileSystem.Dir as Dir

3739-}

core/src/DocTestFileSystemHandle.hs+1−1

@@ -8,7 +8,7 @@

99For APIs that have not been released yet.

1010

11−>>> import qualified Streamly.Internal.Data.Array as Array (writeNUnsafe)

11+>>> import qualified Streamly.Internal.Data.Array as Array (unsafeCreateOf)

1212>>> import qualified Streamly.Internal.Data.Unfold as Unfold (first)

1313>>> import qualified Streamly.Internal.FileSystem.Handle as Handle

1414>>> import qualified Streamly.Internal.System.IO as IO (defaultChunkSize)

core/src/Streamly/Console/Stdio.hs+1−27

@@ -24,33 +24,7 @@ module Streamly.Console.Stdio

2424 -- * Write (stderr)

2525 , writeErr

2626 , writeErrChunks

27−

28− -- * Deprecated

29− , read

30− , readChunks

3127 )

3228where

3329

34−import Control.Monad.IO.Class (MonadIO(..))

35−import Data.Word (Word8)

36−import Streamly.Internal.Data.Array.Type (Array)

37−import Streamly.Internal.Data.Unfold (Unfold)

38−

39−import Streamly.Internal.Console.Stdio hiding (read, readChunks)

40−import Prelude hiding (read)

41−

42−-- Same as 'reader'

43−--

44−-- @since 0.8.0

45−{-# DEPRECATED read "Please use 'reader' instead" #-}

46−{-# INLINE read #-}

47−read :: MonadIO m => Unfold m () Word8

48−read = reader

49−

50−-- Same as 'chunkReader'

51−--

52−-- @since 0.8.0

53−{-# DEPRECATED readChunks "Please use 'chunkReader' instead" #-}

54−{-# INLINE readChunks #-}

55−readChunks :: MonadIO m => Unfold m () (Array Word8)

56−readChunks = chunkReader

30+import Streamly.Internal.Console.Stdio

core/src/Streamly/Data/Array.hs+40−39

@@ -31,28 +31,45 @@ module Streamly.Data.Array

3131 -- * The Array Type

3232 Array

3333

34+ -- * Pinning & Unpinning

35+ -- | Arrays are created unpinned by default unless pinned versions of

36+ -- creation APIs are used. Look for APIs with @pinned@ prefix in

37+ -- "Streamly.Internal.Data.Array" for some unreleased pinned creation APIs.

38+ -- If an array is to be sent to the OS without any further modification

39+ -- then it should be created pinned in the first place instead of pinning

40+ -- it later. Pinning an unpinned array has a copy overhead. OS interfacing

41+ -- APIs create a pinned array directly or convert an unpinned array to

42+ -- pinned array before sending it to the OS.

43+ , pin

44+ , unpin

45+ , isPinned

46+

3447 -- * Construction

3548 -- | When performance matters, the fastest way to generate an array is

36− -- 'writeN'. 'IsList' and 'IsString' instances can be

49+ -- 'createOf'. 'IsList' and 'IsString' instances can be

3750 -- used to conveniently construct arrays from literal values.

3851 -- 'OverloadedLists' extension or 'fromList' can be used to construct an

3952 -- array from a list literal. Similarly, 'OverloadedStrings' extension or

4053 -- 'fromList' can be used to construct an array from a string literal.

4154

42− -- Pure List APIs

55+ -- ** From Stream

56+ , createOf

57+ , create

58+ , writeLastN -- drop old (ring buffer)

59+

60+ -- ** From List

4361 , fromListN

4462 , fromList

4563

46− -- Monadic APIs

47− , writeN -- drop new

48− , write -- full buffer

49− , writeLastN -- drop old (ring buffer)

50−

51− -- * Conversion

64+ -- * To List

5265 -- 'GHC.Exts.toList' from "GHC.Exts" can be used to convert an array to a

5366 -- list.

5467 , toList

5568

69+ -- * To Stream

70+ , read

71+ , readRev

72+

5673 -- * Unfolds

5774 , reader

5875 , readerRev

@@ -66,25 +83,24 @@ module Streamly.Data.Array

6683 -- , (!!)

6784 , getIndex

6885

69− -- * Unbox Type Class

70− , Unbox (..)

86+ -- * Serialization

87+ , pinnedSerialize

88+ , deserialize

7189

72− -- * Pinning & Unpinning

73− , pin

74− , unpin

75− , isPinned

90+ -- * Re-exports

91+ , Unbox (..)

92+ , Serialize(..)

7693

7794 -- * Deprecated

78− , read

79− , readRev

95+ , writeN -- drop new

96+ , write -- full buffer

8097 )

8198where

8299

83100#include "inline.hs"

84101

85−import Streamly.Internal.Data.Array hiding (read, readRev)

86−import Streamly.Internal.Data.Unbox (Unbox (..))

87−import Streamly.Internal.Data.Unfold (Unfold)

102+import Streamly.Internal.Data.Array

103+import Streamly.Internal.Data.MutByteArray (Unbox(..), Serialize(..))

88104

89105import Prelude hiding (read, length)

90106

@@ -102,7 +118,7 @@ import Prelude hiding (read, length)

102118--

103119-- Convert array to stream, and fold the stream:

104120--

105−-- >>> fold f arr = Stream.unfold Array.reader arr & Stream.fold f

121+-- >>> fold f arr = Array.read arr & Stream.fold f

106122-- >>> fold Fold.sum (Array.fromList [1,2,3::Int])

107123-- 6

108124--

@@ -110,7 +126,7 @@ import Prelude hiding (read, length)

110126--

111127-- Convert array to stream, transform, and fold back to array:

112128--

113−-- >>> amap f arr = Stream.unfold Array.reader arr & fmap f & Stream.fold Array.write

129+-- >>> amap f arr = Array.read arr & fmap f & Stream.fold (Array.createOf (Array.length arr))

114130-- >>> amap (+1) (Array.fromList [1,2,3::Int])

115131-- fromList [2,3,4]

116132--

@@ -137,12 +153,11 @@ import Prelude hiding (read, length)

137153-- See the 'fromPureStream' unreleased API to generate an array from an

138154-- Identity stream safely without using MonadIO constraint.

139155--

140−--

141156-- Note that 'Identity' streams can be generalized to IO streams:

142157--

143−-- >>> pure = Stream.fromList [1,2,3] :: Stream Identity Int

144−-- >>> generally = Stream.morphInner (return . runIdentity)

145−-- >>> Stream.fold Array.write (generally pure :: Stream IO Int)

158+-- >>> stream = Stream.fromList [1,2,3] :: Stream Identity Int

159+-- >>> pure2IO s = Stream.morphInner (return . runIdentity) s :: Stream IO Int

160+-- >>> Stream.fold Array.create (pure2IO stream)

146161-- fromList [1,2,3]

147162--

148163-- == Programming Tips

@@ -150,17 +165,3 @@ import Prelude hiding (read, length)

150165-- This module is designed to be imported qualified:

151166--

152167-- >>> import qualified Streamly.Data.Array as Array

153−

154−-- | Same as 'reader'

155−--

156−{-# DEPRECATED read "Please use 'reader' instead" #-}

157−{-# INLINE_NORMAL read #-}

158−read :: (Monad m, Unbox a) => Unfold m (Array a) a

159−read = reader

160−

161−-- | Same as 'readerRev'

162−--

163−{-# DEPRECATED readRev "Please use 'readerRev' instead" #-}

164−{-# INLINE_NORMAL readRev #-}

165−readRev :: (Monad m, Unbox a) => Unfold m (Array a) a

166−readRev = readerRev

core/src/Streamly/Data/Array/Generic.hs+16−13

@@ -15,34 +15,37 @@ module Streamly.Data.Array.Generic

1515 ( Array

1616

1717 -- * Construction

18− , A.fromListN

19− , A.fromList

18+ , fromListN

19+ , fromList

2020

2121 -- MonadicAPIs

22− , A.writeN

23− , A.write

22+ , createOf

23+ , create

2424

2525 -- * Conversion

26− , A.toList

26+ , toList

2727

2828 -- * Streams

29− , A.read

30− , A.readRev

29+ , read

30+ , readRev

3131

3232 -- * Unfolds

33− , A.reader

33+ , reader

3434 -- , A.readerRev

3535

3636 -- * Random Access

37− , A.length

38− , A.getIndex

37+ , length

38+ , getIndex

3939

4040 -- -- * Folding Arrays

4141 -- , A.streamFold

4242 -- , A.fold

43+

44+ -- * Deprecated

45+ , writeN

46+ , write

4347 )

4448where

4549

46−import Streamly.Internal.Data.Array.Generic (Array)

47−

48−import qualified Streamly.Internal.Data.Array.Generic as A

50+import Streamly.Internal.Data.Array.Generic

51+import Prelude hiding (length, read)

core/src/Streamly/Data/Fold.hs+150−118

@@ -10,6 +10,156 @@

1010-- Fast, composable stream consumers with ability to terminate, supporting

1111-- stream fusion.

1212--

13+-- == Using Folds

14+--

15+-- This module provides elementary folds and fold combinators that can be used

16+-- to consume a stream of data and reduce it to a final value, or transform it

17+-- in a stateful manner using scans. A data stream can be reduced into a stream

18+-- of folded data elements by folding segments of the stream. Fold combinators

19+-- can be used to compose multiple folds in parallel or to create a pipeline of

20+-- folds such that the next fold consumes the result of the previous fold. To

21+-- run these folds on a stream see 'Streamly.Data.Stream.fold',

22+-- 'Streamly.Data.Stream.scan', 'Streamly.Data.Stream.postscan',

23+-- 'Streamly.Data.Stream.scanMaybe', 'Streamly.Data.Stream.foldMany' and other

24+-- operations accepting 'Fold' type as argument "Streamly.Data.Stream".

25+--

26+-- == Reducing a Stream

27+--

28+-- A 'Fold' is a consumer of a stream of values. A fold driver (such as

29+-- 'Streamly.Data.Stream.fold') initializes the fold @accumulator@, runs the

30+-- fold @step@ function in a loop, processing the input stream one element at a

31+-- time and accumulating the result. The loop continues until the fold

32+-- terminates, at which point the accumulated result is returned.

33+--

34+-- For example, a 'sum' Fold represents a stream consumer that adds the values

35+-- in the input stream:

36+--

37+-- >>> Stream.fold Fold.sum $ Stream.fromList [1..100]

38+-- 5050

39+--

40+-- Conceptually, a 'Fold' is a data type that mimics a strict left fold

41+-- ('Data.List.foldl'). The above example is similar to a left fold using

42+-- @(+)@ as the step and @0@ as the initial value of the accumulator:

43+--

44+-- >>> Data.List.foldl' (+) 0 [1..100]

45+-- 5050

46+--

47+-- 'Fold's have an early termination capability e.g. the 'one' fold terminates

48+-- after consuming one element:

49+--

50+-- >>> Stream.fold Fold.one $ Stream.fromList [1..]

51+-- Just 1

52+--

53+-- The above example is similar to the following right fold:

54+--

55+-- >>> Prelude.foldr (\x _ -> Just x) Nothing [1..]

56+-- Just 1

57+--

58+-- 'Fold's can be combined together using combinators. For example, to create a

59+-- fold that sums first two elements in a stream:

60+--

61+-- >>> sumTwo = Fold.take 2 Fold.sum

62+-- >>> Stream.fold sumTwo $ Stream.fromList [1..100]

63+-- 3

64+--

65+-- == Parallel Composition

66+--

67+-- Folds can be combined to run in parallel on the same input. For example, to

68+-- compute the average of numbers in a stream without going through the stream

69+-- twice:

70+--

71+-- >>> avg = Fold.teeWith (/) Fold.sum (fmap fromIntegral Fold.length)

72+-- >>> Stream.fold avg $ Stream.fromList [1.0..100.0]

73+-- 50.5

74+--

75+-- Folds can be combined so as to partition the input stream over multiple

76+-- folds. For example, to count even and odd numbers in a stream:

77+--

78+-- >>> split n = if even n then Left n else Right n

79+-- >>> stream = fmap split $ Stream.fromList [1..100]

80+-- >>> countEven = fmap (("Even " ++) . show) Fold.length

81+-- >>> countOdd = fmap (("Odd " ++) . show) Fold.length

82+-- >>> f = Fold.partition countEven countOdd

83+-- >>> Stream.fold f stream

84+-- ("Even 50","Odd 50")

85+--

86+-- == Sequential Composition

87+--

88+-- Terminating folds can be combined to parse the stream serially such that the

89+-- first fold consumes the input until it terminates and the second fold

90+-- consumes the rest of the input until it terminates:

91+--

92+-- >>> f = Fold.splitWith (,) (Fold.take 8 Fold.toList) (Fold.takeEndBy (== '\n') Fold.toList)

93+-- >>> Stream.fold f $ Stream.fromList "header: hello\n"

94+-- ("header: ","hello\n")

95+--

96+-- == Splitting a Stream

97+--

98+-- A 'Fold' can be applied repeatedly on a stream to transform it to a stream

99+-- of fold results. To split a stream on newlines:

100+--

101+-- >>> f = Fold.takeEndBy (== '\n') Fold.toList

102+-- >>> Stream.fold Fold.toList $ Stream.foldMany f $ Stream.fromList "Hello there!\nHow are you\n"

103+-- ["Hello there!\n","How are you\n"]

104+--

105+-- Similarly, we can split the input of a fold too:

106+--

107+-- >>> Stream.fold (Fold.many f Fold.toList) $ Stream.fromList "Hello there!\nHow are you\n"

108+-- ["Hello there!\n","How are you\n"]

109+--

110+-- == Folds vs. Streams

111+--

112+-- We can often use streams or folds to achieve the same goal. However, streams

113+-- are more efficient in composition of producers (e.g.

114+-- 'Data.Stream.append' or 'Data.Stream.mergeBy') whereas folds are

115+-- more efficient in composition of consumers (e.g. 'splitWith', 'partition'

116+-- or 'teeWith').

117+--

118+-- Streams are producers, transformations on streams happen on the output side:

119+--

120+-- >>> :{

121+-- f stream =

122+-- Stream.filter odd stream

123+-- & fmap (+1)

124+-- & Stream.fold Fold.sum

125+-- :}

126+--

127+-- >>> f $ Stream.fromList [1..100 :: Int]

128+-- 2550

129+--

130+-- Folds are stream consumers with an input stream and an output value, stream

131+-- transformations on folds happen on the input side:

132+--

133+-- >>> :{

134+-- f =

135+-- Fold.filter odd

136+-- $ Fold.lmap (+1)

137+-- $ Fold.sum

138+-- :}

139+--

140+-- >>> Stream.fold f $ Stream.fromList [1..100 :: Int]

141+-- 2550

142+--

143+-- Notice the similiarity in the definition of @f@ in both cases, the only

144+-- difference is the composition by @&@ vs @$@ and the use @lmap@ vs @map@, the

145+-- difference is due to output vs input side transformations.

146+--

147+-- == Fusion Limitations

148+--

149+-- Folds support stream fusion for generating loops comparable to the speed of

150+-- C. However, it has some limitations. For fusion to work, the folds must be

151+-- inlined, folds must be statically known and not generated dynamically, folds

152+-- should not be passed recursively.

153+--

154+-- Another limitation is due to the quadratic complexity causing slowdown when

155+-- too many nested compositions are used. Especially, the performance of the

156+-- Applicative instance and splitting operations (e.g. 'splitWith') degrades

157+-- quadratically (O(n^2)) when combined @n@ times, roughly 8 or less sequenced

158+-- operations are fine. For these cases folds can be converted to parsers and

159+-- then used as ParserK.

160+--

161+-- == Experimental APIs

162+--

13163-- Please refer to "Streamly.Internal.Data.Fold" for more functions that have

14164-- not yet been released.

15165

@@ -21,9 +171,6 @@ module Streamly.Data.Fold

21171 --

22172 -- $setup

23173

24− -- * Overview

25− -- $overview

26−

27174 -- * Running A Fold

28175 drive

29176 -- XXX Should we have a stream returning function in fold module?

@@ -258,121 +405,6 @@ import Streamly.Internal.Data.Fold

258405

259406#include "DocTestDataFold.hs"

260407

261−-- $overview

262−--

263−-- A 'Fold' is a consumer of a stream of values. A fold driver (such as

264−-- 'Streamly.Data.Stream.fold') initializes the fold @accumulator@, runs the

265−-- fold @step@ function in a loop, processing the input stream one element at a

266−-- time and accumulating the result. The loop continues until the fold

267−-- terminates, at which point the accumulated result is returned.

268−--

269−-- For example, a 'sum' Fold represents a stream consumer that adds the values

270−-- in the input stream:

271−--

272−-- >>> Stream.fold Fold.sum $ Stream.fromList [1..100]

273−-- 5050

274−--

275−-- Conceptually, a 'Fold' is a data type that mimics a strict left fold

276−-- ('Data.List.foldl'). The above example is similar to a left fold using

277−-- @(+)@ as the step and @0@ as the initial value of the accumulator:

278−--

279−-- >>> Data.List.foldl' (+) 0 [1..100]

280−-- 5050

281−--

282−-- 'Fold's have an early termination capability e.g. the 'one' fold terminates

283−-- after consuming one element:

284−--

285−-- >>> Stream.fold Fold.one $ Stream.fromList [1..]

286−-- Just 1

287−--

288−-- The above example is similar to the following right fold:

289−--

290−-- >>> Prelude.foldr (\x _ -> Just x) Nothing [1..]

291−-- Just 1

292−--

293−-- 'Fold's can be combined together using combinators. For example, to create a

294−-- fold that sums first two elements in a stream:

295−--

296−-- >>> sumTwo = Fold.take 2 Fold.sum

297−-- >>> Stream.fold sumTwo $ Stream.fromList [1..100]

298−-- 3

299−--

300−-- Folds can be combined to run in parallel on the same input. For example, to

301−-- compute the average of numbers in a stream without going through the stream

302−-- twice:

303−--

304−-- >>> avg = Fold.teeWith (/) Fold.sum (fmap fromIntegral Fold.length)

305−-- >>> Stream.fold avg $ Stream.fromList [1.0..100.0]

306−-- 50.5

307−--

308−-- Folds can be combined so as to partition the input stream over multiple

309−-- folds. For example, to count even and odd numbers in a stream:

310−--

311−-- >>> split n = if even n then Left n else Right n

312−-- >>> stream = fmap split $ Stream.fromList [1..100]

313−-- >>> countEven = fmap (("Even " ++) . show) Fold.length

314−-- >>> countOdd = fmap (("Odd " ++) . show) Fold.length

315−-- >>> f = Fold.partition countEven countOdd

316−-- >>> Stream.fold f stream

317−-- ("Even 50","Odd 50")

318−--

319−-- Terminating folds can be combined to parse the stream serially such that the

320−-- first fold consumes the input until it terminates and the second fold

321−-- consumes the rest of the input until it terminates:

322−--

323−-- >>> f = Fold.splitWith (,) (Fold.take 8 Fold.toList) (Fold.takeEndBy (== '\n') Fold.toList)

324−-- >>> Stream.fold f $ Stream.fromList "header: hello\n"

325−-- ("header: ","hello\n")

326−--

327−-- A 'Fold' can be applied repeatedly on a stream to transform it to a stream

328−-- of fold results. To split a stream on newlines:

329−--

330−-- >>> f = Fold.takeEndBy (== '\n') Fold.toList

331−-- >>> Stream.fold Fold.toList $ Stream.foldMany f $ Stream.fromList "Hello there!\nHow are you\n"

332−-- ["Hello there!\n","How are you\n"]

333−--

334−-- Similarly, we can split the input of a fold too:

335−--

336−-- >>> Stream.fold (Fold.many f Fold.toList) $ Stream.fromList "Hello there!\nHow are you\n"

337−-- ["Hello there!\n","How are you\n"]

338−--

339−-- = Folds vs. Streams

340−--

341−-- We can often use streams or folds to achieve the same goal. However, streams

342−-- are more efficient in composition of producers (e.g.

343−-- 'Data.Stream.append' or 'Data.Stream.mergeBy') whereas folds are

344−-- more efficient in composition of consumers (e.g. 'splitWith', 'partition'

345−-- or 'teeWith').

346−--

347−-- Streams are producers, transformations on streams happen on the output side:

348−--

349−-- >>> :{

350−-- f stream =

351−-- Stream.filter odd stream

352−-- & fmap (+1)

353−-- & Stream.fold Fold.sum

354−-- :}

355−--

356−-- >>> f $ Stream.fromList [1..100 :: Int]

357−-- 2550

358−--

359−-- Folds are stream consumers with an input stream and an output value, stream

360−-- transformations on folds happen on the input side:

361−--

362−-- >>> :{

363−-- f =

364−-- Fold.filter odd

365−-- $ Fold.lmap (+1)

366−-- $ Fold.sum

367−-- :}

368−--

369−-- >>> Stream.fold f $ Stream.fromList [1..100 :: Int]

370−-- 2550

371−--

372−-- Notice the similiarity in the definition of @f@ in both cases, the only

373−-- difference is the composition by @&@ vs @$@ and the use @lmap@ vs @map@, the

374−-- difference is due to output vs input side transformations.

375−

376408--------------------------------------------------------------------------------

377409-- Deprecated

378410--------------------------------------------------------------------------------

core/src/Streamly/Data/MutArray.hs+18−12

@@ -32,22 +32,27 @@ module Streamly.Data.MutArray

3232 -- * Construction

3333

3434 -- Uninitialized Arrays

35− , new

36− , pinnedNew

35+ , emptyOf

36+ , pinnedEmptyOf

3737

3838 -- From containers

3939 , fromListN

4040 , fromList

41− , writeN -- drop new

42− , write -- full buffer

41+ , createOf

42+ , create

4343 -- writeLastN

4444

45+ -- * Pinning & Unpinning

46+ , pin

47+ , unpin

48+ , isPinned

49+

4550 -- * Appending elements

4651 , snoc

4752

4853 -- * Appending streams

49− , writeAppendN -- XXX Rename to writeSnocN?

50− , writeAppend -- XXX Rename to writeSnoc?

54+ , appendN

55+ , append

5156

5257 -- * Inplace mutation

5358 , putIndex

@@ -78,16 +83,17 @@ module Streamly.Data.MutArray

7883 -- * Size

7984 , length

8085

81− -- * Unbox Type Class

86+ -- * Re-exports

8287 , Unbox (..)

8388

84− -- * Pinning & Unpinning

85− , pin

86− , unpin

87− , isPinned

88−

8989 -- * Deprecated

9090 , newPinned

91+ , new

92+ , pinnedNew

93+ , writeN

94+ , write

95+ , writeAppendN

96+ , writeAppend

9197 )

9298where

9399

core/src/Streamly/Data/MutArray/Generic.hs+8−3

@@ -23,11 +23,11 @@ module Streamly.Data.MutArray.Generic

2323 MutArray

2424

2525 -- * Construction

26− , new

26+ , emptyOf

2727 , fromListN

2828 , fromList

29− , writeN

30− , write

29+ , createOf

30+ , create

3131

3232 -- * Appending elements

3333 , snoc

@@ -56,6 +56,11 @@ module Streamly.Data.MutArray.Generic

5656

5757 -- * Size

5858 , length

59+

60+ -- * Deprecated

61+ , new

62+ , writeN

63+ , write

5964 )

6065where

6166

core/src/Streamly/Data/MutByteArray.hsnew file+106

@@ -0,0 +1,106 @@

1+-- |

2+-- Module : Streamly.Data.MutByteArray

4+-- License : BSD3-3-Clause

5+-- Maintainer : streamly@composewell.com

6+-- Portability : GHC

7+--

8+-- A low level byte Array type 'MutByteArray', along with type classes 'Unbox'

9+-- and 'Serialize' for fast binary serialization and deserialization of Haskell

10+-- values. Serialization, deserialization performance is similar to, and in

11+-- some cases many times better than the store package. Conceptually, the

12+-- 'Serialize' type class works in the same way as store.

13+--

14+-- Serialize instances are configurable to use constructor names (see

15+-- 'Streamly.Internal.Data.MutByteArray.encodeConstrNames'), record field names (see

16+-- 'Streamly.Internal.Data.MutByteArray.encodeRecordFields') instead of binary

17+-- encoded values. This is an experimental feature which allows JSON like

18+-- properties with faster speed. For example, you can change the order of

19+-- constructors or record fields without affecting serialized value.

20+--

21+-- Higher level unboxed array modules "Streamly.Data.Array" and

22+-- "Streamly.Data.MutArray" are built on top of this module. Unboxed arrays are

23+-- essentially serialized Haskell values. Array modules provide higher level

24+-- serialization routines like 'Streamly.Internal.Data.Array.pinnedSerialize'

25+-- and 'Streamly.Internal.Data.Array.deserialize' from the

26+-- "Streamly.Internal.Data.Array" module.

27+--

28+-- == Mutable Byte Array

29+--

30+-- 'MutByteArray' is a primitive mutable array in the IO monad. 'Unbox' and

31+-- 'Serialize' type classes use this primitive array to serialize data to and

32+-- deserialize it from. This array is used to build higher level unboxed

33+-- array types 'Streamly.Data.MutArray.MutArray' and 'Streamly.Data.Array.Array'.

34+--

35+-- == Using Unbox

36+--

37+-- The 'Unbox' type class is simple and used to serialize non-recursive fixed

38+-- size data types. This type class is primarily used to implement unboxed

39+-- arrays. Unboxed arrays are just a sequence of serialized fixed length

40+-- Haskell data types. Instances of this type class can be derived using

41+-- 'Generic' or template haskell based deriving functions provided in this

42+-- module.

43+--

44+-- Writing a data type to an array using the array creation routines in

45+-- "Streamly.Data.Array" or "Streamly.Data.MutArray" (e.g. @writeN@ or

46+-- @fromListN@), serializes the type to the array. Similarly, reading the data

47+-- type from the array deserializes it. You can also serialize and deserialize

48+-- directly to and from a 'MutByteArray', using the type class methods.

49+--

50+-- == Using Serialize

51+--

52+-- The 'Serialize' type class is a superset of the 'Unbox' type class, it can

53+-- serialize variable length data types as well e.g. Haskell lists. Use

54+-- 'deriveSerialize' to derive the instances of the type class automatically

55+-- and then use the type class methods to serialize and deserialize to and from

56+-- a 'MutByteArray'.

57+--

58+-- See 'Streamly.Internal.Data.Array.pinnedSerialize' and

59+-- 'Streamly.Internal.Data.Array.deserialize' for 'Array' type based

60+-- serialization.

61+--

62+-- == Comparing serialized values

63+--

64+-- When using the `Unbox` type class the same value may result in differing

65+-- serialized bytes because of unused uninitialized data in case of sum types.

66+-- Therefore, byte comparison of serialized values is not reliable.

67+--

68+-- However, the 'Serialize' type class guarantees that the serialized values

69+-- are always exactly the same and byte comparison of serialized is reliable.

70+--

71+module Streamly.Data.MutByteArray

72+ (

73+

74+ -- * Mutable Byte Array

75+ -- | The standard way to read from or write to a 'MutByteArray' is by using

76+ -- the 'Unbox' or 'Serialize' type class methods.

77+ MutByteArray

78+ , isPinned

79+ , pin

80+ , unpin

81+ , new

82+ , pinnedNew

83+

84+ -- * Unbox

85+ , Unbox(..)

86+ , deriveUnbox

87+

88+ -- * Serialize

89+ , Serialize(..)

90+

91+ -- ** Instance Config

92+ , SerializeConfig

93+ , inlineAddSizeTo

94+ , inlineSerializeAt

95+ , inlineDeserializeAt

96+

97+ -- ** Instance Deriving

98+ , deriveSerialize

99+ , deriveSerializeWith

100+ ) where

101+

102+--------------------------------------------------------------------------------

103+-- Imports

104+--------------------------------------------------------------------------------

105+

106+import Streamly.Internal.Data.MutByteArray

core/src/Streamly/Data/Parser.hs+130−60

@@ -7,12 +7,137 @@

77-- Stability : pre-release

88-- Portability : GHC

99--

10−-- Fast, composable stream consumers with ability to terminate, backtrack and

11−-- fail, supporting stream fusion. Parsers are a natural extension of

12−-- "Streamly.Data.Fold". Parsers and folds can be interconverted.

10+-- Parsers are stream consumers like folds with the following differences:

1311--

14−-- Please refer to "Streamly.Internal.Data.Parser" for functions that have

15−-- not yet been released.

12+-- * folds cannot fail but parsers can fail and backtrack.

13+-- * folds can be composed as a Tee but parsers cannot.

14+-- * folds can be used for scanning but parsers cannot.

15+-- * folds can be converted to parsers.

16+--

17+-- This module implements parsers with stream fusion which compile to efficient

18+-- loops comparable to the speed of C.

19+--

20+-- == Using Parsers

21+--

22+-- This module provides elementary parsers and parser combinators that can be

23+-- used to parse a stream of data. Additionally, all the folds from the

24+-- "Streamly.Data.Fold" module can be converted to parsers using 'fromFold'.

25+-- All the parsing functionality provided by popular parsing libraries, and

26+-- more is available. Also see "Streamly.Unicode.Parser" module for Char stream

27+-- parsers.

28+--

29+-- A data stream can be transformed to a stream of parsed data elements. Parser

30+-- combinators can be used to create a pipeline of folds or parsers such that

31+-- the next fold or parser consumes the result of the previous parser. See

32+-- 'Streamly.Data.Stream.parse' and 'Streamly.Data.Stream.parseMany' to run

33+-- these parsers on a stream.

34+--

35+-- == Parser vs ParserK

36+--

37+-- There are two functionally equivalent parsing modules,

38+-- "Streamly.Data.Parser" (this module) and "Streamly.Data.ParserK". The latter

39+-- is a CPS based wrapper over the former, and can be used for parsing in

40+-- general. "Streamly.Data.Parser" enables stream fusion and should be

41+-- preferred over "Streamly.Data.ParserK" for high performance stream parsing

42+-- use cases. However, there are a few cases where this module is not

43+-- suitable and ParserK should be used instead.

44+--

45+-- For static fusion, parser combinators have to use strict pattern matching on

46+-- arguments of type Parser. This leads to infinte loop when a parser is

47+-- defined recursively, due to strict evaluation of the recursive call. For

48+-- example, the following implementation loops infinitely because of the

49+-- recursive use of parser @p@ in the @*>@ combinator:

50+--

51+-- >>> import Streamly.Data.Parser (Parser)

52+-- >>> import qualified Streamly.Data.Fold as Fold

53+-- >>> import qualified Streamly.Data.Parser as Parser

54+-- >>> import qualified Streamly.Data.Stream as Stream

55+-- >>> import Control.Applicative ((<|>))

56+--

57+-- >>> :{

58+-- >>> p :: Monad m => Parser Char m String

59+-- >>> p = Parser.satisfy (== '(') *> p <|> Parser.fromFold Fold.toList

60+-- >>> :}

61+--

62+-- Use ParserK when recursive use is required:

63+--

64+-- >>> import Streamly.Data.ParserK (ParserK)

65+-- >>> import qualified Streamly.Data.StreamK as StreamK

66+-- >>> import qualified Streamly.Internal.Data.StreamK as StreamK (parse)

67+-- >>> import qualified Streamly.Internal.Data.ParserK as ParserK (adapt)

68+--

69+-- >>> :{

70+-- >>> p :: Monad m => ParserK Char m String

71+-- >>> p = ParserK.adapt (Parser.satisfy (== '(')) *> p <|> ParserK.adapt (Parser.fromFold Fold.toList)

72+-- >>> :}

73+--

74+-- >>> StreamK.parse p $ StreamK.fromStream $ Stream.fromList "hello"

75+-- Right "hello"

76+--

77+-- For this reason Applicative, Alternative or Monad compositions with

78+-- recursion cannot be used with the 'Parser' type. Alternative type class based

79+-- operations like 'asum' and Alternative based generic parser combinators use

80+-- recursion. Similarly, Applicative type class based operations like

81+-- 'Prelude.sequence' use recursion. Custom implementations of many such

82+-- operations are provided in this module (e.g. 'some', 'many'), and those

83+-- should be used instead.

84+--

85+-- Another limitation of Parser type is due to the quadratic complexity causing

86+-- slowdown when too many nested compositions are used. Especially Applicative,

87+-- Monad, Alternative instances, and sequenced parsing operations (e.g. nested

88+-- 'one', and 'splitWith') degrade the performance quadratically (O(n^2)) when

89+-- combined @n@ times, roughly 8 or less sequenced parsers are fine. READ THE

90+-- DOCS OF APPLICATIVE, MONAD AND ALTERNATIVE INSTANCES.

91+--

92+-- == Streaming Parsers

93+--

94+-- With 'Streamly.Data.ParserK.ParserK' you can use the generic Alternative

95+-- type class based parsers from the

96+-- <https://hackage.haskell.org/package/parser-combinators parser-combinators>

97+-- library or similar. However, we recommend that you use the equivalent

98+-- functionality from this module for better performance and for streaming

99+-- behavior.

100+--

101+-- Firstly, the combinators in this module are faster due to stream fusion.

102+-- Secondly, these are streaming in nature as the results can be passed

103+-- directly to other stream consumers (folds or parsers). The Alternative type

104+-- class based parsers would end up buffering all the results in lists before

105+-- they can be consumed.

106+--

107+-- When recursion or heavy nesting is needed use ParserK.

108+--

109+-- == Error Reporting

110+--

111+-- These parsers do not report the error context (e.g. line number or column).

112+-- This may be supported in future.

113+--

114+-- == Monad Transformer Stack

115+--

116+-- 'MonadTrans' instance is not provided. If the 'Parser' type is the top most

117+-- layer (which should be the case almost always) you can just use 'fromEffect'

118+-- to execute the lower layer monad effects.

119+--

120+-- == Parser vs ParserK Implementation

121+--

122+-- The 'Parser' type represents a stream consumer by composing state as data

123+-- which enables stream fusion. Stream fusion generates a tight loop without

124+-- any constructor allocations between the stages, providing C like performance

125+-- for the loop. Stream fusion works when multiple functions are combined in a

126+-- pipeline statically. Therefore, the operations in this module must be

127+-- inlined and must not be used recursively to allow for stream fusion.

128+--

129+-- The 'ParserK' type represents a stream consumer by composing function calls,

130+-- therefore, a function call overhead is incurred at each composition. It is

131+-- quite fast in general but may be a few times slower than a fused parser.

132+-- However, it allows for scalable dynamic composition especially parsers can

133+-- be used in recursive calls. Using the 'ParserK' type operations like

134+-- 'splitWith' provide linear (O(n)) performance with respect to the number of

135+-- compositions.

136+--

137+-- == Experimental APIs

138+--

139+-- Please refer to "Streamly.Internal.Data.Parser" for functions that have not

140+-- yet been released.

16141--

17142module Streamly.Data.Parser

18143 (

@@ -22,9 +147,6 @@ module Streamly.Data.Parser

22147 --

23148 -- $setup

24149

25− -- * Overview

26− -- $overview

27−

28150 -- * Parser Type

29151 Parser

30152

@@ -123,55 +245,3 @@ import Streamly.Internal.Data.Parser

123245import Prelude hiding (dropWhile, takeWhile, filter)

124246

125247#include "DocTestDataParser.hs"

126−

127−-- $overview

128−--

129−-- Several combinators in this module can be many times faster than CPS based

130−-- parsers because of stream fusion. For example,

131−-- 'Streamly.Internal.Data.Parser.many' combinator in this module is much

132−-- faster than the 'Control.Applicative.many' combinator of

133−-- 'Control.Applicative.Alternative' type class used by CPS based parsers.

134−--

135−-- The use of 'Alternative' type class, in parsers has another drawback.

136−-- Alternative based parsers use plain Haskell lists to collect the results. In

137−-- a strict Monad like IO, the results are necessarily buffered before they can

138−-- be consumed. This may not perform optimally in streaming applications

139−-- processing large amounts of data. Equivalent combinators in this module can

140−-- consume the results of parsing using a 'Fold' or another parser, thus

141−-- providing a scalable and composable consumer.

142−--

143−-- Note that these parsers do not report the error context (e.g. line number or

144−-- column). This may be supported in future.

145−--

146−-- mtl instances are not provided. If the 'Parser' type is the top most layer

147−-- (which should be the case almost always) you can just use 'fromEffect' to

148−-- execute the lower layer monad effects.

149−--

150−-- == Performance Notes

151−--

152−-- The 'Parser' type represents a stream consumer by composing state as data

153−-- which enables stream fusion. Stream fusion generates a tight loop without

154−-- any constructor allocations between the stages, providing C like performance

155−-- for the loop. Stream fusion works when multiple functions are combined in a

156−-- pipeline statically. Therefore, the operations in this module must be

157−-- inlined and must not be used recursively to allow for stream fusion. Note

158−-- that operations like 'sequence', and 'asum' that compose pasrers using

159−-- recursion should be avoided with these parsers. You can use these with the

160−-- 'ParserK' module instead.

161−--

162−-- Using the 'Parser' type, parsing operations like 'one', 'splitWith' etc.

163−-- degrade quadratically (O(n^2)) when combined many times. If you need to

164−-- combine these operations, say more than 8 times in a single loop, then you

165−-- should consider using the continuation style parser type 'ParserK' instead.

166−-- Also, if you need to use these operations in a recursive loop you should use

167−-- 'ParserK' instead.

168−--

169−-- The 'ParserK' type represents a stream consumer by composing function calls,

170−-- therefore, a function call overhead is incurred at each composition. It is

171−-- quite fast in general but may be a few times slower than a fused parser.

172−-- However, it allows for scalable dynamic composition especially parsers can

173−-- be used in recursive calls. Using the 'ParserK' type operations like

174−-- 'splitWith' provide linear (O(n)) performance with respect to the number of

175−-- compositions..

176−--

177−-- 'Parser' and 'ParserK' types can be interconverted.

core/src/Streamly/Data/ParserK.hs+89−3

@@ -1,3 +1,4 @@

1+{-# LANGUAGE CPP #-}

12-- |

23-- Module : Streamly.Data.ParserK

@@ -6,19 +7,102 @@

67-- Stability : pre-release

78-- Portability : GHC

89--

9−-- Parsers using Continuation Passing Style (CPS). See notes in

10−-- "Streamly.Data.Parser" module to know when to use this module.

10+-- See the general notes about parsing in the "Streamly.Data.Parser" module.

11+-- This module implements a using Continuation Passing Style (CPS) wrapper over

12+-- the "Streamly.Data.Parser" module. It is as fast or faster than attoparsec.

1113--

12−-- To run a 'ParserK' use 'Streamly.Data.StreamK.parseChunks'.

14+-- == Parser vs ParserK

15+--

16+-- 'ParserK' is preferred over 'Streamly.Data.Parser.Parser' when extensive

17+-- applicative, alternative and monadic composition is required, or when

18+-- recursive or dynamic composition of parsers is required. The

19+-- 'Streamly.Data.Parser.Parser' type fuses statically and creates efficient

20+-- loops whereas 'ParserK' uses function call based composition and has

21+-- comparatively larger runtime overhead but it is better suited to the

22+-- specific use cases mentioned above. 'ParserK' also allows to efficient parse

23+-- a stream of arrays, it can also break the input stream into a parse result

24+-- and remaining stream so that the stream can be parsed independently in

25+-- segments.

26+--

27+-- == Using ParserK

28+--

29+-- All the parsers from the "Streamly.Data.Parser" module can be adapted to

30+-- ParserK using the 'Streamly.Data.ParserK.adaptC',

31+-- 'Streamly.Internal.Data.ParserK.adapt', and

32+-- 'Streamly.Internal.Data.ParserK.adaptCG' combinators.

33+--

34+-- 'Streamly.Data.StreamK.parseChunks' runs a parser on a stream of unboxed

35+-- arrays, this is the preferred and most efficient way to parse chunked input.

36+-- The more general 'Streamly.Data.StreamK.parseBreakChunks' function returns

37+-- the remaining stream as well along with the parse result. There are

38+-- 'Streamly.Internal.Data.StreamK.parseChunksGeneric',

39+-- 'Streamly.Internal.Data.StreamK.parseBreakChunksGeneric' as well to run

40+-- parsers on boxed arrays. 'Streamly.Internal.Data.StreamK.parse',

41+-- 'Streamly.Internal.Data.StreamK.parseBreak' run parsers on a stream of

42+-- individual elements instead of stream of arrays.

43+--

44+-- == Monadic Composition

45+--

46+-- Monad composition can be used for lookbehind parsers, we can dynamically

47+-- compose new parsers based on the results of the previously parsed values.

48+--

49+-- If we have to parse "a9" or "9a" but not "99" or "aa" we can use the

50+-- following non-monadic, backtracking parser:

51+--

52+-- >>> digits p1 p2 = ((:) <$> p1 <*> ((:) <$> p2 <*> pure []))

53+-- >>> :{

54+-- backtracking :: Monad m => ParserK Char m String

55+-- backtracking = ParserK.adapt $

56+-- digits (Parser.satisfy isDigit) (Parser.satisfy isAlpha)

57+-- <|>

58+-- digits (Parser.satisfy isAlpha) (Parser.satisfy isDigit)

59+-- :}

60+--

61+-- We know that if the first parse resulted in a digit at the first place then

62+-- the second parse is going to fail. However, we waste that information and

63+-- parse the first character again in the second parse only to know that it is

64+-- not an alphabetic char. By using lookbehind in a 'Monad' composition we can

65+-- avoid redundant work:

66+--

67+-- >>> data DigitOrAlpha = Digit Char | Alpha Char

68+--

69+-- >>> :{

70+-- lookbehind :: Monad m => ParserK Char m String

71+-- lookbehind = do

72+-- x1 <- ParserK.adapt $

73+-- Digit <$> Parser.satisfy isDigit

74+-- <|> Alpha <$> Parser.satisfy isAlpha

75+-- -- Note: the parse depends on what we parsed already

76+-- x2 <- ParserK.adapt $

77+-- case x1 of

78+-- Digit _ -> Parser.satisfy isAlpha

79+-- Alpha _ -> Parser.satisfy isDigit

80+-- return $ case x1 of

81+-- Digit x -> [x,x2]

82+-- Alpha x -> [x,x2]

83+-- :}

84+--

85+-- == Experimental APIs

86+--

87+-- Please refer to "Streamly.Internal.Data.ParserK" for functions that have

88+-- not yet been released.

1389--

1490module Streamly.Data.ParserK

1591 (

92+ -- * Setup

93+ -- | To execute the code examples provided in this module in ghci, please

94+ -- run the following commands first.

95+ --

96+ -- $setup

97+

1698 -- * Parser Type

1799 ParserK

18100

19101 -- * Parsers

20102 -- ** Conversions

103+ , adapt

21104 , adaptC

105+ , adaptCG

22106 -- , toParser

23107

24108 -- ** Without Input

@@ -41,6 +125,8 @@ import qualified Streamly.Internal.Data.Parser as ParserD

41125

42126import Streamly.Internal.Data.ParserK.Type

43127

128+#include "DocTestDataParserK.hs"

129+

44130{-# DEPRECATED fromFold "Please use \"ParserK.adaptC . Parser.fromFold\" instead." #-}

45131{-# INLINE fromFold #-}

46132fromFold :: (MonadIO m, Unbox a) => Fold m a b -> ParserK (Array a) m b

core/src/Streamly/Data/Serialize.hsdeleted−50

@@ -1,50 +0,0 @@

1−-- |

2−-- Module : Streamly.Data.Serialize

4−-- License : BSD3-3-Clause

5−-- Maintainer : streamly@composewell.com

6−-- Portability : GHC

7−--

8−module Streamly.Data.Serialize

9− (

10−

11− -- * Types

12− MutableByteArray

13−

14− -- * Unbox

15− , Unbox(..)

16− , deriveUnbox

17− , deriveUnboxWith

18−

19− -- * Serialize

20− , Serialize(..)

21− , deriveSerialize

22− , deriveSerializeWith

23−

24− -- * Encoding

25− , encode

26− , pinnedEncode

27−

28− -- * Decoding

29− , decode

30− ) where

31−

32−--------------------------------------------------------------------------------

33−-- Imports

34−--------------------------------------------------------------------------------

35−

36−import Streamly.Internal.Data.Serialize

37− ( Serialize(..)

38− , decode

39− , encode

40− , pinnedEncode

41− )

42−import Streamly.Internal.Data.Serialize.TH

43− ( deriveSerialize

44− , deriveSerializeWith

45− )

46−import Streamly.Internal.Data.Unbox

47− ( MutableByteArray

48− , Unbox(..)

49− )

50−import Streamly.Internal.Data.Unbox.TH (deriveUnbox, deriveUnboxWith)

core/src/Streamly/Data/Stream.hs+131−36

@@ -70,6 +70,15 @@ module Streamly.Data.Stream

7070

7171 -- ** From Values

7272 -- | Generate a monadic stream from a seed value or values.

73+ --

74+ -- Non-empty pure streams can be cycled using repeat and unfold:

75+ --

76+ -- >>> cycle = Stream.unfoldMany Unfold.fromList . Stream.repeat

77+ --

78+ -- Non-empty StreamK can be cycled using append:

79+ --

80+ -- >>> cycle xs = let ys = xs `StreamK.append` ys in ys

81+ --

7382 , fromPure

7483 , fromEffect

7584 , repeat

@@ -212,21 +221,57 @@ module Streamly.Data.Stream

212221 -- operations like mapping a function over the stream.

213222 , foldrM

214223 , foldr

224+ -- foldr1

215225

216226 -- ** Specific Folds

217− -- | Usually you can use the folds in "Streamly.Data.Fold". However, some

218− -- folds that may be commonly used or may have an edge in performance in

219− -- some cases are provided here.

220− --

221− -- Useful idioms:

227+ -- | Streams are folded using folds in "Streamly.Data.Fold". Here are some

228+ -- idioms and equivalents of Data.List APIs using folds:

222229 --

223230 -- >>> foldlM' f a = Stream.fold (Fold.foldlM' f a)

224− -- >>> foldl1 f = Stream.fold (Fold.foldl1' f)

231+ -- >>> foldl1' f = Stream.fold (Fold.foldl1' f)

225232 -- >>> foldl' f a = Stream.fold (Fold.foldl' f a)

226233 -- >>> drain = Stream.fold Fold.drain

227234 -- >>> mapM_ f = Stream.fold (Fold.drainMapM f)

228235 -- >>> length = Stream.fold Fold.length

236+ -- >>> genericLength = Stream.fold Fold.lengthGeneric

229237 -- >>> head = Stream.fold Fold.one

238+ -- >>> last = Stream.fold Fold.latest

239+ -- >>> null = Stream.fold Fold.null

240+ -- >>> and = Stream.fold Fold.and

241+ -- >>> or = Stream.fold Fold.or

242+ -- >>> any p = Stream.fold (Fold.any p)

243+ -- >>> all p = Stream.fold (Fold.all p)

244+ -- >>> sum = Stream.fold Fold.sum

245+ -- >>> product = Stream.fold Fold.product

246+ -- >>> maximum = Stream.fold Fold.maximum

247+ -- >>> maximumBy cmp = Stream.fold (Fold.maximumBy cmp)

248+ -- >>> minimum = Stream.fold Fold.minimum

249+ -- >>> minimumBy cmp = Stream.fold (Fold.minimumBy cmp)

250+ -- >>> elem x = Stream.fold (Fold.elem x)

251+ -- >>> notElem x = Stream.fold (Fold.notElem x)

252+ -- >>> lookup x = Stream.fold (Fold.lookup x)

253+ -- >>> find p = Stream.fold (Fold.find p)

254+ -- >>> (!?) i = Stream.fold (Fold.index i)

255+ -- >>> genericIndex i = Stream.fold (Fold.indexGeneric i)

256+ -- >>> elemIndex x = Stream.fold (Fold.elemIndex x)

257+ -- >>> findIndex p = Stream.fold (Fold.findIndex p)

258+ --

259+ -- Some equivalents of Data.List APIs from the Stream module:

260+ --

261+ -- >>> head = fmap (fmap fst) . Stream.uncons

262+ -- >>> tail = fmap (fmap snd) . Stream.uncons

263+ -- >>> tail = Stream.tail -- unreleased API

264+ -- >>> init = Stream.init -- unreleased API

265+ --

266+ -- A Stream based toList fold implementation is provided below because it

267+ -- has a better performance compared to the fold.

268+

269+ -- Functions in Data.List, missing here:

270+ -- unsnoc = Stream.parseBreak (Parser.init Fold.toList)

271+ -- genericTake

272+ -- genericDrop

273+ -- genericSplitAt

274+ -- genericReplicate

230275 , toList

231276

232277 -- * Mapping

@@ -280,6 +325,15 @@ module Streamly.Data.Stream

280325 , scan

281326 , postscan

282327 -- XXX postscan1 can be implemented using Monoids or Refolds.

328+ -- The following scans from Data.List are not provided.

329+ -- XXX scanl

330+ -- XXX scanl1

331+ -- XXX scanr

332+ -- XXX scanr1

333+ -- XXX mapAccumL

334+ -- XXX mapAccumR

335+ -- XXX inits

336+ -- XXX tails

283337

284338 -- ** Specific scans

285339 -- Indexing can be considered as a special type of zipping where we zip a

@@ -288,6 +342,8 @@ module Streamly.Data.Stream

288342

289343 -- * Insertion

290344 -- | Add elements to the stream.

345+ --

346+ -- >>> insert = Stream.insertBy compare

291347

292348 -- Inserting elements is a special case of interleaving/merging streams.

293349 , insertBy

@@ -323,12 +379,15 @@ module Streamly.Data.Stream

323379 -- "Streamly.Internal.Data.Fold" can be used along with 'scanMaybe' to

324380 -- perform stateful filtering operations in general.

325381 --

326− -- Useful idioms:

382+ -- Idioms and equivalents of Data.List APIs:

327383 --

328384 -- >>> deleteBy cmp x = Stream.scanMaybe (Fold.deleteBy cmp x)

385+ -- >>> deleteBy = Stream.deleteBy -- unreleased API

386+ -- >>> delete = deleteBy (==)

329387 -- >>> findIndices p = Stream.scanMaybe (Fold.findIndices p)

330388 -- >>> elemIndices a = findIndices (== a)

331389 -- >>> uniq = Stream.scanMaybe (Fold.uniqBy (==))

390+ -- >>> partition p = Stream.fold (Fold.partition Fold.toList Fold.toList) . fmap (if p then Left else Right)

332391 , scanMaybe

333392 , take

334393 , takeWhile

@@ -336,6 +395,8 @@ module Streamly.Data.Stream

336395 , drop

337396 , dropWhile

338397 , dropWhileM

398+ -- XXX write to an array in reverse and then read in reverse

399+ -- > dropWhileEnd = reverse . dropWhile p . reverse

339400

340401 -- XXX These are available as scans in folds. We need to check the

341402 -- performance though. If these are common and we need convenient stream

@@ -372,6 +433,9 @@ module Streamly.Data.Stream

372433 -- >>> s = s1 `Stream.append` s2

373434

374435 -- ** Appending

436+ -- | Equivalent of Data.List append:

437+ --

438+ -- >>> (++) = Stream.append

375439 , append

376440

377441 -- ** Interleaving

@@ -382,8 +446,14 @@ module Streamly.Data.Stream

382446 , mergeByM

383447

384448 -- ** Zipping

449+ -- | Idioms and equivalents of Data.List APIs:

450+ --

451+ -- >>> zip = Stream.zipWith (,)

452+ -- >>> unzip = Stream.fold (Fold.unzip Fold.toList Fold.toList)

385453 , zipWith

386454 , zipWithM

455+ -- XXX zipWith3,4,5,6,7

456+ -- XXX unzip3,4,5,6,7

387457 -- , ZipStream (..)

388458

389459 -- ** Cross Product

@@ -416,48 +486,89 @@ module Streamly.Data.Stream

416486 -- One dimension loops are just a special case of nested loops. For

417487 -- example, 'concatMap' can degenerate to a simple map operation:

418488 --

419− -- > map f m = S.concatMap (\x -> S.fromPure (f x)) m

489+ -- >>> map f m = Stream.concatMap (\x -> Stream.fromPure (f x)) m

420490 --

421491 -- Similarly, 'concatMap' can perform filtering by mapping an element to a

422492 -- 'nil' stream:

423493 --

424− -- > filter p m = S.concatMap (\x -> if p x then S.fromPure x else S.nil) m

494+ -- >>> filter p m = Stream.concatMap (\x -> if p x then Stream.fromPure x else Stream.nil) m

425495 --

496+ -- Idioms and equivalents of Data.List APIs:

497+ --

498+ -- >>> concat = Stream.concatMap id

499+ -- >>> unlines = Stream.interposeSuffix '\n'

500+ -- >>> unlines = Stream.intercalateSuffix Unfold.fromList "\n"

501+ -- >>> unwords = Stream.interpose ' '

502+ -- >>> unwords = Stream.intercalate Unfold.fromList " "

426503

427504 , concatEffect

428505 , concatMap

429506 , concatMapM

430507

431508 -- * Repeated Fold

432− -- | Useful idioms:

509+ -- | Idioms and equivalents of Data.List APIs:

433510 --

434− -- >>> splitWithSuffix p f = Stream.foldMany (Fold.takeEndBy p f)

435− -- >>> splitOnSuffix p f = Stream.foldMany (Fold.takeEndBy_ p f)

436− -- >>> groupsBy eq f = Stream.parseMany (Parser.groupBy eq f)

437− -- >>> groupsByRolling eq f = Stream.parseMany (Parser.groupByRolling eq f)

438− -- >>> groupsOf n f = Stream.foldMany (Fold.take n f)

511+ -- >>> groupsOf n = Stream.foldMany (Fold.take n Fold.toList)

512+ -- >>> groupBy eq = Stream.groupsWhile eq Fold.toList

513+ -- >>> groupBy eq = Stream.parseMany (Parser.groupBy eq Fold.toList)

514+ -- >>> groupsByRolling eq = Stream.parseMany (Parser.groupByRolling eq Fold.toList)

515+ -- >>> groups = groupBy (==)

439516 , foldMany -- XXX Rename to foldRepeat

440517 , groupsOf

441518 , parseMany

442519

443520 -- * Splitting

521+ -- | Idioms and equivalents of Data.List APIs:

522+ --

523+ -- >>> splitWithSuffix p f = Stream.foldMany (Fold.takeEndBy p f)

524+ -- >>> splitOnSuffix p f = Stream.foldMany (Fold.takeEndBy_ p f)

525+ -- >>> lines = splitOnSuffix (== '\n')

526+ -- >>> words = Stream.wordsBy isSpace

527+ -- >>> splitAt n = Stream.fold (Fold.splitAt n Fold.toList Fold.toList)

528+ -- >>> span p = Parser.splitWith (,) (Parser.takeWhile p Fold.toList) (Parser.fromFold Fold.toList)

529+ -- >>> break p = span (not . p)

444530 , splitOn

445531 , wordsBy

446532

447533 -- * Buffered Operations

448534 -- | Operations that require buffering of the stream.

449535 -- Reverse is essentially a left fold followed by an unfold.

536+ --

537+ -- Idioms and equivalents of Data.List APIs:

538+ --

539+ -- >>> nub = Stream.fold Fold.toList . Stream.scanMaybe Fold.nub

540+ -- >>> nub = Stream.ordNub -- unreleased API

541+ -- >>> sortBy = StreamK.sortBy

542+ -- >>> sortOn f = StreamK.sortOn -- unreleased API

543+ -- >>> deleteFirstsBy = Stream.deleteFirstsBy -- unreleased

544+ -- >>> (\\) = Stream.deleteFirstsBy (==) -- unreleased

545+ -- >>> intersectBy = Stream.intersectBy -- unreleased

546+ -- >>> intersect = Stream.intersectBy (==) -- unreleased

547+ -- >>> unionBy = Stream.unionBy -- unreleased

548+ -- >>> union = Stream.unionBy (==) -- unreleased

549+ --

450550 , reverse

551+ -- XXX transpose: write the streams to arrays and then stream transposed.

552+ -- XXX subsequences

553+ -- XXX permutations

554+ -- , nub

555+ -- , ordNub

556+ -- , nubBy

451557

452558 -- * Multi-Stream folds

453559 -- | Operations that consume multiple streams at the same time.

454560 , eqBy

455561 , cmpBy

456562 , isPrefixOf

563+ , isInfixOf

564+ -- , isSuffixOf

565+ -- , isSuffixOfUnbox

457566 , isSubsequenceOf

458567

459568 -- trimming sequences

460− , stripPrefix

569+ -- , stripPrefix

570+ -- , stripSuffix

571+ -- , stripSuffixUnbox

461572

462573 -- Exceptions and resource management depend on the "exceptions" package

463574 -- XXX We can have IO Stream operations not depending on "exceptions"

@@ -511,25 +622,20 @@ module Streamly.Data.Stream

511622 , runReaderT

512623 , runStateT

513624

625+ -- XXX Arrays could be different types, therefore, this should be in

626+ -- specific array module. Or maybe we should abstract over array types.

514627 -- * Stream of Arrays

515628 , Array.chunksOf

516− , chunksSplitOn

517629 )

518630where

519631

520−import Control.Monad.IO.Class (MonadIO)

521−import Data.Word (Word8)

522−import Streamly.Data.Array (Array)

523632import Streamly.Internal.Data.Stream

524633import Prelude

525634 hiding (filter, drop, dropWhile, take, takeWhile, zipWith, foldr,

526− foldl, map, mapM, mapM_, sequence, all, any, sum, product, elem,

527− notElem, maximum, minimum, head, last, tail, length, null,

528− reverse, iterate, init, and, or, lookup, foldr1, (!!),

529− scanl, scanl1, repeat, replicate, concatMap, span)

635+ mapM, sequence, reverse, iterate, foldr1, repeat, replicate,

636+ concatMap)

530637

531638import qualified Streamly.Internal.Data.Array.Type as Array

532−import qualified Streamly.Internal.Data.Stream.Chunked as ArrayStream

533639

534640#include "DocTestDataStream.hs"

535641

@@ -638,14 +744,3 @@ import qualified Streamly.Internal.Data.Stream.Chunked as ArrayStream

638744--

639745-- 'Stream' and 'StreamK' types can be interconverted. See

640746-- "Streamly.Data.StreamK" module for conversion operations.

641−

642−-- | Split a stream of arrays on a given separator byte, dropping the separator

643−-- and coalescing all the arrays between two separators into a single array.

644−--

645−{-# INLINE chunksSplitOn #-}

646−chunksSplitOn

647− :: (MonadIO m)

648− => Word8

649− -> Stream m (Array Word8)

650− -> Stream m (Array Word8)

651−chunksSplitOn = ArrayStream.splitOn

core/src/Streamly/Data/StreamK.hs+2−1

@@ -95,7 +95,8 @@ module Streamly.Data.StreamK

9595 -- , foldBreak

9696

9797 -- ** Parsing

98− -- , parseBreak

98+ , parse

99+ , parseBreak

99100 , parseBreakChunks

100101 , parseChunks

101102

core/src/Streamly/Data/Unfold.hs+2

@@ -65,6 +65,8 @@ module Streamly.Data.Unfold

6565 -- ** Mapping on Input

6666 , lmap

6767 , lmapM

68+ , first

69+ , second

6870

6971 -- ** Mapping on Output

7072 , mapM

core/src/Streamly/FileSystem/Handle.hs+7−30

@@ -1,5 +1,4 @@

11{-# LANGUAGE CPP #-}

2−#include "inline.hs"

43-- |

54-- Module : Streamly.FileSystem.Handle

@@ -82,13 +81,13 @@ module Streamly.FileSystem.Handle

8281 -- position of the file handle. The stream ends as soon as EOF is

8382 -- encountered.

8483

85− -- -- *** Streams

86− -- , read

87− -- , readWith

88− -- , readChunks

89− -- , readChunksWith

84+ -- *** Streams

85+ , read

86+ , readWith

87+ , readChunks

88+ , readChunksWith

9089

91− -- -- *** Unfolds

90+ -- *** Unfolds

9291 , reader

9392 , readerWith

9493 , chunkReader

@@ -105,36 +104,14 @@ module Streamly.FileSystem.Handle

105104 , writeChunks

106105

107106 -- * Deprecated

108− , read

109107 , readWithBufferOf

110− , readChunks

111108 , readChunksWithBufferOf

112109 , writeChunksWithBufferOf

113110 , writeWithBufferOf

114111 )

115112where

116113

117−import Control.Monad.IO.Class (MonadIO(..))

118−import Data.Word (Word8)

119−import Streamly.Internal.Data.Array.Type (Array)

120−import Streamly.Internal.Data.Unfold.Type (Unfold)

121−import System.IO (Handle)

122−

123−import Streamly.Internal.FileSystem.Handle hiding (read, readChunks)

114+import Streamly.Internal.FileSystem.Handle

124115import Prelude hiding (read)

125116

126117#include "DocTestFileSystemHandle.hs"

127−

128−-- | Same as 'reader'

129−--

130−{-# DEPRECATED read "Please use 'reader' instead" #-}

131−{-# INLINE read #-}

132−read :: MonadIO m => Unfold m Handle Word8

133−read = reader

134−

135−-- | Same as 'chunkReader'

136−--

137−{-# DEPRECATED readChunks "Please use 'chunkReader' instead" #-}

138−{-# INLINE readChunks #-}

139−readChunks :: MonadIO m => Unfold m Handle (Array Word8)

140−readChunks = chunkReader

core/src/Streamly/Internal/Data/Array.hs+608−135

@@ -2,7 +2,6 @@

22-- |

33-- Module : Streamly.Internal.Data.Array

5−--

65-- License : BSD3

76-- Maintainer : streamly@composewell.com

87-- Stability : experimental

@@ -20,33 +19,21 @@ module Streamly.Internal.Data.Array

2019 module Streamly.Internal.Data.Array.Type

2120

2221 -- * Construction

23− -- Stream Folds

24− , fromStreamN

25− , fromStream

26−

2722 -- Monadic Folds

2823 , writeLastN

2924

30− -- * Elimination

31− -- ** Conversion

32−

33− -- ** Unfolds

34− , reader

35− , readerUnsafe

36− , producer -- experimental

37−

3825 -- * Random Access

3926 -- , (!!)

4027 , getIndex

4128 , getIndexRev

42− , last -- XXX getIndexLast?

43− , getIndices

44− , getIndicesFromThenTo

29+ , last -- XXX getLastIndex?

4530 -- , getIndicesFrom -- read from a given position to the end of file

4631 -- , getIndicesUpto -- read from beginning up to the given position

4732 -- , getIndicesFromTo

4833 -- , getIndicesFromRev -- read from a given position to the beginning of file

4934 -- , getIndicesUptoRev -- read from end to the given position in file

35+ , indexReader

36+ , indexReaderFromThenTo

5037

5138 -- * Size

5239 , null

@@ -54,6 +41,8 @@ module Streamly.Internal.Data.Array

5441 -- * Search

5542 , binarySearch

5643 , findIndicesOf

44+ -- getIndicesOf

45+ , indexFinder -- see splitOn

5746 -- , findIndexOf

5847 -- , find

5948

@@ -66,51 +55,89 @@ module Streamly.Internal.Data.Array

6655 -- * Subarrays

6756 , getSliceUnsafe

6857 -- , getSlice

69− , genSlicesFromLen

70− , getSlicesFromLen

58+ , sliceIndexerFromLen

59+ , slicerFromLen

7160 , splitOn

7261

7362 -- * Streaming Operations

7463 , streamTransform

7564

76− -- ** Folding

65+ -- * Folding

7766 , streamFold

7867 , fold

68+

69+ -- * Stream of Arrays

70+

71+ -- XXX these are probably not very useful to have in this module as we can

72+ -- express these idiomatically using streams.

73+ , interpose

74+ , interposeSuffix

75+ , intercalateSuffix

76+

77+ , compactLE

78+ , pinnedCompactLE

79+ , compactOnByte

80+ , compactOnByteSuffix

81+

82+ , foldBreakChunks

83+ , foldChunks

84+ , foldBreakChunksK

85+ , parseBreakChunksK

86+

87+ -- * Serialization

88+ , encodeAs

89+ , serialize

90+ , pinnedSerialize

91+ , deserialize

92+

93+ -- * Deprecated

94+ , genSlicesFromLen

95+ , getSlicesFromLen

96+ , getIndices

7997 )

8098where

8199

100+#include "assert.hs"

82101#include "inline.hs"

83102#include "ArrayMacros.h"

84103

85−import Control.Exception (assert)

86−import Control.Monad (when)

87104import Control.Monad.IO.Class (MonadIO(..))

105+-- import Data.Bifunctor (first)

106+-- import Data.Either (fromRight)

88107import Data.Functor.Identity (Identity)

89108import Data.Proxy (Proxy(..))

90109import Data.Word (Word8)

91110import Foreign.C.String (CString)

92111import Foreign.Ptr (castPtr)

93112import Foreign.Storable (Storable)

113+import GHC.Types (SPEC(..))

94114import Streamly.Internal.Data.Unbox (Unbox(..))

95115import Prelude hiding (length, null, last, map, (!!), read, concat)

96116

117+import Streamly.Internal.Data.MutByteArray.Type (PinnedState(..), MutByteArray)

118+import Streamly.Internal.Data.Serialize.Type (Serialize)

97119import Streamly.Internal.Data.Fold.Type (Fold(..))

98−import Streamly.Internal.Data.Producer.Type (Producer(..))

99−import Streamly.Internal.Data.Stream (Stream)

100−import Streamly.Internal.Data.Tuple.Strict (Tuple3Fused'(..))

120+import Streamly.Internal.Data.Parser (Parser(..), Initial(..), ParseError(..))

121+import Streamly.Internal.Data.Stream (Stream(..))

122+import Streamly.Internal.Data.StreamK (StreamK)

123+import Streamly.Internal.Data.SVar.Type (adaptState, defState)

124+import Streamly.Internal.Data.Tuple.Strict (Tuple'(..), Tuple3Fused'(..))

101125import Streamly.Internal.Data.Unfold.Type (Unfold(..))

102126import Streamly.Internal.System.IO (unsafeInlineIO)

103127

104−import qualified Streamly.Internal.Data.MutArray.Type as MA

128+import qualified Streamly.Internal.Data.Fold.Type as Fold

129+import qualified Streamly.Internal.Data.Serialize.Type as Serialize

130+import qualified Streamly.Internal.Data.MutByteArray.Type as MBA

105131import qualified Streamly.Internal.Data.MutArray as MA

106−import qualified Streamly.Internal.Data.Array.Type as A

107132import qualified Streamly.Internal.Data.Fold as FL

108−import qualified Streamly.Internal.Data.Producer.Type as Producer

109−import qualified Streamly.Internal.Data.Producer as Producer

110133import qualified Streamly.Internal.Data.Ring as RB

134+import qualified Streamly.Internal.Data.Parser as Parser

135+-- import qualified Streamly.Internal.Data.ParserK as ParserK

111136import qualified Streamly.Internal.Data.Stream as D

112137import qualified Streamly.Internal.Data.Stream as Stream

138+import qualified Streamly.Internal.Data.StreamK as StreamK

113139import qualified Streamly.Internal.Data.Unfold as Unfold

140+import qualified Prelude

114141

115142import Streamly.Internal.Data.Array.Type

116143

@@ -139,82 +166,10 @@ import Streamly.Internal.Data.Array.Type

139166-- if we use one array for each chunk. This is still significant to add

140167-- pressure to GC.

141168

142−-------------------------------------------------------------------------------

143−-- Construction

144−-------------------------------------------------------------------------------

145−

146−-- | Create an 'Array' from the first N elements of a stream. The array is

147−-- allocated to size N, if the stream terminates before N elements then the

148−-- array may hold less than N elements.

149−--

150−-- /Pre-release/

151−{-# INLINE fromStreamN #-}

152−fromStreamN :: (MonadIO m, Unbox a) => Int -> Stream m a -> m (Array a)

153−fromStreamN n m = do

154− when (n < 0) $ error "writeN: negative write count specified"

155− A.fromStreamDN n m

156−

157−-- | Create an 'Array' from a stream. This is useful when we want to create a

158−-- single array from a stream of unknown size. 'writeN' is at least twice

159−-- as efficient when the size is already known.

160−--

161−-- Note that if the input stream is too large memory allocation for the array

162−-- may fail. When the stream size is not known, `chunksOf` followed by

163−-- processing of indvidual arrays in the resulting stream should be preferred.

164−--

165−-- /Pre-release/

166−{-# INLINE fromStream #-}

167−fromStream :: (MonadIO m, Unbox a) => Stream m a -> m (Array a)

168−fromStream = Stream.fold A.write

169−-- write m = A.fromStreamD $ D.fromStreamK m

170−

171169-------------------------------------------------------------------------------

172170-- Elimination

173171-------------------------------------------------------------------------------

174172

175−{-# INLINE_NORMAL producer #-}

176−producer :: forall m a. (Monad m, Unbox a) => Producer m (Array a) a

177−producer =

178− Producer.translate A.unsafeThaw A.unsafeFreeze

179− $ MA.producerWith (return . unsafeInlineIO)

180−

181−-- | Unfold an array into a stream.

182−--

183−{-# INLINE_NORMAL reader #-}

184−reader :: forall m a. (Monad m, Unbox a) => Unfold m (Array a) a

185−reader = Producer.simplify producer

186−

187−-- | Unfold an array into a stream, does not check the end of the array, the

188−-- user is responsible for terminating the stream within the array bounds. For

189−-- high performance application where the end condition can be determined by

190−-- a terminating fold.

191−--

192−-- Written in the hope that it may be faster than "read", however, in the case

193−-- for which this was written, "read" proves to be faster even though the core

194−-- generated with unsafeRead looks simpler.

195−--

196−-- /Pre-release/

197−--

198−{-# INLINE_NORMAL readerUnsafe #-}

199−readerUnsafe :: forall m a. (Monad m, Unbox a) => Unfold m (Array a) a

200−readerUnsafe = Unfold step inject

201− where

202−

203− inject (Array contents start end) =

204− return (ArrayUnsafe contents end start)

205−

206− {-# INLINE_LATE step #-}

207− step (ArrayUnsafe contents end p) = do

208− -- unsafeInlineIO allows us to run this in Identity monad for pure

209− -- toList/foldr case which makes them much faster due to not

210− -- accumulating the list and fusing better with the pure consumers.

211− --

212− -- This should be safe as the array contents are guaranteed to be

213− -- evaluated/written to before we peek at them.

214− let !x = unsafeInlineIO $ peekByteIndex p contents

215− let !p1 = INDEX_NEXT(p,a)

216− return $ D.Yield x (ArrayUnsafe contents end p1)

217−

218173-- |

219174--

220175-- >>> null arr = Array.byteLength arr == 0

@@ -222,7 +177,7 @@ readerUnsafe = Unfold step inject

222177-- /Pre-release/

223178{-# INLINE null #-}

224179null :: Array a -> Bool

225−null arr = A.byteLength arr == 0

180+null arr = byteLength arr == 0

226181

227182-- | Like 'getIndex' but indexes the array in reverse from the end.

228183--

@@ -234,7 +189,7 @@ getIndexRev i arr =

234189 $ do

235190 let elemPtr = RINDEX_OF(arrEnd arr, i, a)

236191 if i >= 0 && elemPtr >= arrStart arr

237− then Just <$> peekByteIndex elemPtr (arrContents arr)

192+ then Just <$> peekAt elemPtr (arrContents arr)

238193 else return Nothing

239194

240195-- |

@@ -250,6 +205,8 @@ last = getIndexRev 0

250205-- Folds with Array as the container

251206-------------------------------------------------------------------------------

252207

208+-- XXX We should generate this from Ring.

209+

253210-- | @writeLastN n@ folds a maximum of @n@ elements from the end of the input

254211-- stream to an 'Array'.

255212--

@@ -258,7 +215,7 @@ writeLastN ::

258215 (Storable a, Unbox a, MonadIO m) => Int -> Fold m a (Array a)

259216writeLastN n

260217 | n <= 0 = fmap (const mempty) FL.drain

261− | otherwise = A.unsafeFreeze <$> Fold step initial done done

218+ | otherwise = unsafeFreeze <$> Fold step initial done done

262219

263220 where

264221

@@ -271,11 +228,9 @@ writeLastN n

271228 in fmap f $ liftIO $ RB.new n

272229

273230 done (Tuple3Fused' rb rh i) = do

274− arr <- liftIO $ MA.new n

275− foldFunc i rh snoc' arr rb

276−

277− -- XXX We should write a read unfold for ring.

278− snoc' b a = liftIO $ MA.snocUnsafe b a

231+ arr <- MA.new n

232+ -- XXX We should write a read unfold for ring.

233+ foldFunc i rh MA.snocUnsafe arr rb

279234

280235 foldFunc i

281236 | i < n = RB.unsafeFoldRingM

@@ -301,19 +256,26 @@ binarySearch = undefined

301256-- compared to streams by using SIMD instructions.

302257-- We can also return a bit array instead.

303258

259+-- Can use SIMD.

260+

304261-- | Perform a linear search to find all the indices where a given element is

305262-- present in an array.

306263--

307264-- /Unimplemented/

308−findIndicesOf :: (a -> Bool) -> Unfold Identity (Array a) Int

309−findIndicesOf = undefined

265+indexFinder :: (a -> Bool) -> Unfold Identity (Array a) Int

266+indexFinder = undefined

267+

268+-- |

269+-- /Unimplemented/

270+findIndicesOf :: (a -> Bool) -> Array a -> Stream Identity Int

271+findIndicesOf p = Stream.unfold (indexFinder p)

310272

311273{-

312274findIndexOf :: (a -> Bool) -> Array a -> Maybe Int

313−findIndexOf p = Unfold.fold Fold.one . Stream.unfold (findIndicesOf p)

275+findIndexOf p = Unfold.fold Fold.one . Stream.unfold (indexFinder p)

314276

315277find :: (a -> Bool) -> Array a -> Bool

316−find = Unfold.fold Fold.null . Stream.unfold (findIndicesOf p)

278+find = Unfold.fold Fold.null . Stream.unfold (indexFinder p)

317279-}

318280

319281-------------------------------------------------------------------------------

@@ -355,29 +317,45 @@ splitOn :: (Monad m, Unbox a) =>

355317 (a -> Bool) -> Array a -> Stream m (Array a)

356318splitOn predicate arr =

357319 fmap (\(i, len) -> getSliceUnsafe i len arr)

358− $ D.sliceOnSuffix predicate (A.toStreamD arr)

320+ $ D.indexOnSuffix predicate (read arr)

321+

322+{-# INLINE sliceIndexerFromLen #-}

323+sliceIndexerFromLen :: forall m a. (Monad m, Unbox a)

324+ => Int -- ^ from index

325+ -> Int -- ^ length of the slice

326+ -> Unfold m (Array a) (Int, Int)

327+sliceIndexerFromLen from len =

328+ Unfold.lmap unsafeThaw (MA.sliceIndexerFromLen from len)

359329

330+{-# DEPRECATED genSlicesFromLen "Please use sliceIndexerFromLen instead." #-}

360331{-# INLINE genSlicesFromLen #-}

361332genSlicesFromLen :: forall m a. (Monad m, Unbox a)

362333 => Int -- ^ from index

363334 -> Int -- ^ length of the slice

364335 -> Unfold m (Array a) (Int, Int)

365−genSlicesFromLen from len =

366− Unfold.lmap A.unsafeThaw (MA.genSlicesFromLen from len)

336+genSlicesFromLen = sliceIndexerFromLen

367337

368338-- | Generate a stream of slices of specified length from an array, starting

369339-- from the supplied array index. The last slice may be shorter than the

370340-- requested length.

371341--

372342-- /Pre-release//

343+{-# INLINE slicerFromLen #-}

344+slicerFromLen :: forall m a. (Monad m, Unbox a)

345+ => Int -- ^ from index

346+ -> Int -- ^ length of the slice

347+ -> Unfold m (Array a) (Array a)

348+slicerFromLen from len =

349+ fmap unsafeFreeze

350+ $ Unfold.lmap unsafeThaw (MA.slicerFromLen from len)

351+

352+{-# DEPRECATED getSlicesFromLen "Please use slicerFromLen instead." #-}

373353{-# INLINE getSlicesFromLen #-}

374354getSlicesFromLen :: forall m a. (Monad m, Unbox a)

375355 => Int -- ^ from index

376356 -> Int -- ^ length of the slice

377357 -> Unfold m (Array a) (Array a)

378−getSlicesFromLen from len =

379− fmap A.unsafeFreeze

380− $ Unfold.lmap A.unsafeThaw (MA.getSlicesFromLen from len)

358+getSlicesFromLen = slicerFromLen

381359

382360-------------------------------------------------------------------------------

383361-- Random reads and writes

@@ -395,7 +373,7 @@ getIndex i arr =

395373 $ do

396374 let elemPtr = INDEX_OF(arrStart arr, i, a)

397375 if i >= 0 && INDEX_VALID(elemPtr, arrEnd arr, a)

398− then Just <$> peekByteIndex elemPtr (arrContents arr)

376+ then Just <$> peekAt elemPtr (arrContents arr)

399377 else return Nothing

400378

401379-- | Given a stream of array indices, read the elements on those indices from

@@ -407,19 +385,25 @@ getIndex i arr =

407385-- @

408386-- read =

409387-- let u = lmap (\arr -> (0, length arr - 1)) Unfold.enumerateFromTo

410−-- in Unfold.lmap f (getIndices arr)

388+-- in Unfold.lmap f (indexReader arr)

411389--

412390-- readRev =

413391-- let i = length arr - 1

414−-- in Unfold.lmap f (getIndicesFromThenTo i (i - 1) 0)

392+-- in Unfold.lmap f (indexReaderFromThenTo i (i - 1) 0)

415393-- @

416394--

417395-- /Pre-release/

396+{-# INLINE indexReader #-}

397+indexReader :: (Monad m, Unbox a) => Stream m Int -> Unfold m (Array a) a

398+indexReader m =

399+ let unf = MA.indexReaderWith (return . unsafeInlineIO) m

400+ in Unfold.lmap unsafeThaw unf

401+

402+-- XXX DO NOT REMOVE, change the signature to use Stream instead of unfold

403+{-# DEPRECATED getIndices "Please use getIndices instead." #-}

418404{-# INLINE getIndices #-}

419405getIndices :: (Monad m, Unbox a) => Stream m Int -> Unfold m (Array a) a

420−getIndices m =

421− let unf = MA.getIndicesD (return . unsafeInlineIO) m

422− in Unfold.lmap A.unsafeThaw unf

406+getIndices = indexReader

423407

424408-- | Unfolds @(from, then, to, array)@ generating a finite stream whose first

425409-- element is the array value from the index @from@ and the successive elements

@@ -435,9 +419,9 @@ getIndices m =

435419-- @

436420--

437421-- /Unimplemented/

438−{-# INLINE getIndicesFromThenTo #-}

439−getIndicesFromThenTo :: Unfold m (Int, Int, Int, Array a) a

440−getIndicesFromThenTo = undefined

422+{-# INLINE indexReaderFromThenTo #-}

423+indexReaderFromThenTo :: Unfold m (Int, Int, Int, Array a) a

424+indexReaderFromThenTo = undefined

441425

442426-------------------------------------------------------------------------------

443427-- Transform via stream operations

@@ -454,7 +438,7 @@ getIndicesFromThenTo = undefined

454438{-# INLINE runPipe #-}

455439runPipe :: (MonadIO m, Unbox a, Unbox b)

456440 => Pipe m a b -> Array a -> m (Array b)

457−runPipe f arr = P.runPipe (toArrayMinChunk (length arr)) $ f (A.read arr)

441+runPipe f arr = P.runPipe (toArrayMinChunk (length arr)) $ f (read arr)

458442-}

459443

460444-- XXX For transformations that cannot change the number of elements e.g. "map"

@@ -468,7 +452,7 @@ runPipe f arr = P.runPipe (toArrayMinChunk (length arr)) $ f (A.read arr)

468452streamTransform :: forall m a b. (MonadIO m, Unbox a, Unbox b)

469453 => (Stream m a -> Stream m b) -> Array a -> m (Array b)

470454streamTransform f arr =

471− Stream.fold (A.writeWith (length arr)) $ f (A.read arr)

455+ Stream.fold (createWith (length arr)) $ f (read arr)

472456

473457-------------------------------------------------------------------------------

474458-- Casts

@@ -502,7 +486,7 @@ asBytes = castUnsafe

502486--

503487cast :: forall a b. (Unbox b) => Array a -> Maybe (Array b)

504488cast arr =

505− let len = A.byteLength arr

489+ let len = byteLength arr

506490 r = len `mod` SIZE_OF(b)

507491 in if r /= 0

508492 then Nothing

@@ -520,9 +504,9 @@ cast arr =

520504--

521505asCStringUnsafe :: Array a -> (CString -> IO b) -> IO b

522506asCStringUnsafe arr act = do

523− let arr1 = asBytes arr <> A.fromList [0]

524− -- asPtrUnsafe makes sure the array is pinned

525− asPtrUnsafe arr1 $ \ptr -> act (castPtr ptr)

507+ let arr1 = asBytes arr <> fromList [0]

508+ -- unsafePinnedAsPtr makes sure the array is pinned

509+ unsafePinnedAsPtr arr1 $ \ptr -> act (castPtr ptr)

526510

527511-------------------------------------------------------------------------------

528512-- Folds

@@ -535,11 +519,500 @@ asCStringUnsafe arr act = do

535519-- /Pre-release/

536520{-# INLINE fold #-}

537521fold :: forall m a b. (Monad m, Unbox a) => Fold m a b -> Array a -> m b

538−fold f arr = Stream.fold f (A.read arr)

522+fold f arr = Stream.fold f (read arr)

539523

540524-- | Fold an array using a stream fold operation.

541525--

542526-- /Pre-release/

543527{-# INLINE streamFold #-}

544528streamFold :: (Monad m, Unbox a) => (Stream m a -> m b) -> Array a -> m b

545−streamFold f arr = f (A.read arr)

529+streamFold f arr = f (read arr)

530+

531+--------------------------------------------------------------------------------

532+-- Serialization

533+--------------------------------------------------------------------------------

534+

535+{-# INLINE encodeAs #-}

536+encodeAs :: forall a. Serialize a => PinnedState -> a -> Array Word8

537+encodeAs ps a =

538+ unsafeInlineIO $ do

539+ let len = Serialize.addSizeTo 0 a

540+ mbarr <- MBA.newBytesAs ps len

541+ off <- Serialize.serializeAt 0 mbarr a

542+ assertM(len == off)

543+ pure $ Array mbarr 0 off

544+

545+-- |

546+-- Properties:

547+-- 1. Identity: @deserialize . serialize == id@

548+-- 2. Encoded equivalence: @serialize a == serialize a@

549+{-# INLINE serialize #-}

550+serialize :: Serialize a => a -> Array Word8

551+serialize = encodeAs Unpinned

552+

553+-- | Serialize a Haskell type to a pinned byte array. The array is allocated

554+-- using pinned memory so that it can be used directly in OS APIs for writing

555+-- to file or sending over the network.

556+--

557+-- Properties:

558+-- 1. Identity: @deserialize . pinnedSerialize == id@

559+-- 2. Encoded equivalence: @pinnedSerialize a == pinnedSerialize a@

560+{-# INLINE pinnedSerialize #-}

561+pinnedSerialize :: Serialize a => a -> Array Word8

562+pinnedSerialize = encodeAs Pinned

563+

564+-- | Decode a Haskell type from a byte array containing its serialized

565+-- representation.

566+{-# INLINE deserialize #-}

567+deserialize :: Serialize a => Array Word8 -> a

568+deserialize arr@(Array {..}) = unsafeInlineIO $ do

569+ let lenArr = length arr

570+ (off, val) <-

571+ Serialize.deserializeAt arrStart arrContents (arrStart + lenArr)

572+ assertM(off == arrStart + lenArr)

573+ pure val

574+

575+-------------------------------------------------------------------------------

576+-- Streams of Arrays

577+-------------------------------------------------------------------------------

578+

579+-- TODO: efficiently compare two streams of arrays. Two streams can have chunks

580+-- of different sizes, we can handle that in the stream comparison abstraction.

581+-- This could be useful e.g. to fast compare whether two files differ.

582+

583+-- | Insert the given element between arrays and flatten.

584+--

585+-- >>> interpose x = Stream.interpose x Array.reader

586+--

587+{-# INLINE interpose #-}

588+interpose :: (Monad m, Unbox a) => a -> Stream m (Array a) -> Stream m a

589+interpose x = D.interpose x reader

590+

591+data FlattenState s =

592+ OuterLoop s

593+ | InnerLoop s !MutByteArray !Int !Int

594+

595+-- | Insert the given element after each array and flatten. This is similar to

596+-- unlines.

597+--

598+-- >>> interposeSuffix x = Stream.interposeSuffix x Array.reader

599+--

600+{-# INLINE_NORMAL interposeSuffix #-}

601+interposeSuffix :: forall m a. (Monad m, Unbox a)

602+ => a -> Stream m (Array a) -> Stream m a

603+-- This does not require MonadIO constraint.

604+-- interposeSuffix x = D.interposeSuffix x reader

605+interposeSuffix sep (D.Stream step state) = D.Stream step' (OuterLoop state)

606+

607+ where

608+

609+ {-# INLINE_LATE step' #-}

610+ step' gst (OuterLoop st) = do

611+ r <- step (adaptState gst) st

612+ return $ case r of

613+ D.Yield Array{..} s ->

614+ D.Skip (InnerLoop s arrContents arrStart arrEnd)

615+ D.Skip s -> D.Skip (OuterLoop s)

616+ D.Stop -> D.Stop

617+

618+ step' _ (InnerLoop st _ p end) | p == end =

619+ return $ D.Yield sep $ OuterLoop st

620+

621+ step' _ (InnerLoop st contents p end) = do

622+ x <- pure $ unsafeInlineIO $ peekAt p contents

623+ return $ D.Yield x (InnerLoop st contents (INDEX_NEXT(p,a)) end)

624+

625+-- | Insert the given array after each array and flatten.

626+--

627+-- >>> intercalateSuffix = Stream.intercalateSuffix Array.reader

628+--

629+{-# INLINE intercalateSuffix #-}

630+intercalateSuffix :: (Monad m, Unbox a)

631+ => Array a -> Stream m (Array a) -> Stream m a

632+intercalateSuffix = D.intercalateSuffix reader

633+

634+-- | @compactLE n@ coalesces adjacent arrays in the input stream

635+-- only if the combined size would be less than or equal to n.

636+--

637+-- Generates unpinned arrays irrespective of the pinning status of input

638+-- arrays.

639+{-# INLINE_NORMAL compactLE #-}

640+compactLE :: (MonadIO m, Unbox a)

641+ => Int -> Stream m (Array a) -> Stream m (Array a)

642+compactLE n stream =

643+ D.map unsafeFreeze $ MA.compactLE n $ D.map unsafeThaw stream

644+

645+-- | Pinned version of 'compactLE'.

646+{-# INLINE_NORMAL pinnedCompactLE #-}

647+pinnedCompactLE :: (MonadIO m, Unbox a)

648+ => Int -> Stream m (Array a) -> Stream m (Array a)

649+pinnedCompactLE n stream =

650+ D.map unsafeFreeze $ MA.pinnedCompactLE n $ D.map unsafeThaw stream

651+

652+-- | Split a stream of arrays on a given separator byte, dropping the separator

653+-- and coalescing all the arrays between two separators into a single array.

654+--

655+{-# INLINE compactOnByte #-}

656+compactOnByte

657+ :: (MonadIO m)

658+ => Word8

659+ -> Stream m (Array Word8)

660+ -> Stream m (Array Word8)

661+compactOnByte byte =

662+ fmap unsafeFreeze . MA.compactOnByte byte . fmap unsafeThaw

663+

664+-- | Like 'compactOnByte' considers the separator in suffix position instead of

665+-- infix position.

666+{-# INLINE compactOnByteSuffix #-}

667+compactOnByteSuffix

668+ :: (MonadIO m)

669+ => Word8

670+ -> Stream m (Array Word8)

671+ -> Stream m (Array Word8)

672+compactOnByteSuffix byte =

673+ fmap unsafeFreeze . MA.compactOnByteSuffix byte . fmap unsafeThaw

674+

675+-------------------------------------------------------------------------------

676+-- Folding Streams of Arrays

677+-------------------------------------------------------------------------------

678+

679+-- XXX This should not be used for breaking a stream as the D.cons used in

680+-- reconstructing the stream could be very bad for performance. This can only

681+-- be useful in folding without breaking.

682+{-# INLINE_NORMAL foldBreakChunks #-}

683+foldBreakChunks :: forall m a b. (MonadIO m, Unbox a) =>

684+ Fold m a b -> Stream m (Array a) -> m (b, Stream m (Array a))

685+foldBreakChunks (Fold fstep initial _ final) stream@(Stream step state) = do

686+ res <- initial

687+ case res of

688+ Fold.Partial fs -> go SPEC state fs

689+ Fold.Done fb -> return $! (fb, stream)

690+

691+ where

692+

693+ {-# INLINE go #-}

694+ go !_ st !fs = do

695+ r <- step defState st

696+ case r of

697+ Stream.Yield (Array contents start end) s ->

698+ let fp = Tuple' end contents

699+ in goArray SPEC s fp start fs

700+ Stream.Skip s -> go SPEC s fs

701+ Stream.Stop -> do

702+ b <- final fs

703+ return (b, D.nil)

704+

705+ goArray !_ s (Tuple' end _) !cur !fs

706+ | cur == end = do

707+ go SPEC s fs

708+ goArray !_ st fp@(Tuple' end contents) !cur !fs = do

709+ x <- liftIO $ peekAt cur contents

710+ res <- fstep fs x

711+ let next = INDEX_NEXT(cur,a)

712+ case res of

713+ Fold.Done b -> do

714+ let arr = Array contents next end

715+ return $! (b, D.cons arr (D.Stream step st))

716+ Fold.Partial fs1 -> goArray SPEC st fp next fs1

717+

718+-- This may be more robust wrt fusion compared to unfoldMany?

719+

720+-- | Fold a stream of arrays using a 'Fold'. This is equivalent to the

721+-- following:

722+--

723+-- >>> foldChunks f = Stream.fold f . Stream.unfoldMany Array.reader

724+--

725+foldChunks :: (MonadIO m, Unbox a) => Fold m a b -> Stream m (Array a) -> m b

726+foldChunks f s = fmap fst (foldBreakChunks f s)

727+-- foldStream f = Stream.fold f . Stream.unfoldMany reader

728+

729+-- | Fold a stream of arrays using a 'Fold' and return the remaining stream.

730+--

731+-- The following alternative to this function allows composing the fold using

732+-- the parser Monad:

733+--

734+-- @

735+-- foldBreakStreamK f s =

736+-- fmap (first (fromRight undefined))

737+-- $ StreamK.parseBreakChunks (ParserK.adaptC (Parser.fromFold f)) s

738+-- @

739+--

740+-- We can compare perf and remove this one or define it in terms of that.

741+--

742+foldBreakChunksK :: forall m a b. (MonadIO m, Unbox a) =>

743+ Fold m a b -> StreamK m (Array a) -> m (b, StreamK m (Array a))

744+{-

745+foldBreakChunksK f s =

746+ fmap (first (fromRight undefined))

747+ $ StreamK.parseBreakChunks (ParserK.adaptC (Parser.fromFold f)) s

748+-}

749+foldBreakChunksK (Fold fstep initial _ final) stream = do

750+ res <- initial

751+ case res of

752+ Fold.Partial fs -> go fs stream

753+ Fold.Done fb -> return (fb, stream)

754+

755+ where

756+

757+ {-# INLINE go #-}

758+ go !fs st = do

759+ let stop = (, StreamK.nil) <$> final fs

760+ single a = yieldk a StreamK.nil

761+ yieldk (Array contents start end) r =

762+ let fp = Tuple' end contents

763+ in goArray fs r fp start

764+ in StreamK.foldStream defState yieldk single stop st

765+

766+ goArray !fs st (Tuple' end _) !cur

767+ | cur == end = do

768+ go fs st

769+ goArray !fs st fp@(Tuple' end contents) !cur = do

770+ x <- liftIO $ peekAt cur contents

771+ res <- fstep fs x

772+ let next = INDEX_NEXT(cur,a)

773+ case res of

774+ Fold.Done b -> do

775+ let arr = Array contents next end

776+ return $! (b, StreamK.cons arr st)

777+ Fold.Partial fs1 -> goArray fs1 st fp next

778+

779+{-

780+-- This can be generalized to any type provided it can be unfolded to a stream

781+-- and it can be combined using a semigroup operation.

782+--

783+{-# INLINE_NORMAL parseBreakD #-}

784+parseBreakD ::

785+ forall m a b. (MonadIO m, MonadThrow m, Unbox a)

786+ => PRD.Parser a m b

787+ -> D.Stream m (Array.Array a)

788+ -> m (b, D.Stream m (Array.Array a))

789+parseBreakD

790+ (PRD.Parser pstep initial extract) stream@(D.Stream step state) = do

791+

792+ res <- initial

793+ case res of

794+ PRD.IPartial s -> go SPEC state (List []) s

795+ PRD.IDone b -> return (b, stream)

796+ PRD.IError err -> throwM $ ParseError err

797+

798+ where

799+

800+ -- "backBuf" contains last few items in the stream that we may have to

801+ -- backtrack to.

802+ --

803+ -- XXX currently we are using a dumb list based approach for backtracking

804+ -- buffer. This can be replaced by a sliding/ring buffer using Data.Array.

805+ -- That will allow us more efficient random back and forth movement.

806+ go !_ st backBuf !pst = do

807+ r <- step defState st

808+ case r of

809+ D.Yield (Array contents start end) s ->

810+ gobuf SPEC s backBuf

811+ (Tuple' end contents) start pst

812+ D.Skip s -> go SPEC s backBuf pst

813+ D.Stop -> do

814+ b <- extract pst

815+ return (b, D.nil)

816+

817+ -- Use strictness on "cur" to keep it unboxed

818+ gobuf !_ s backBuf (Tuple' end _) !cur !pst

819+ | cur == end = do

820+ go SPEC s backBuf pst

821+ gobuf !_ s backBuf fp@(Tuple' end contents) !cur !pst = do

822+ x <- liftIO $ peekByteIndex contents cur

823+ pRes <- pstep pst x

824+ let next = INDEX_NEXT(cur,a)

825+ case pRes of

826+ PR.Partial 0 pst1 ->

827+ gobuf SPEC s (List []) fp next pst1

828+ PR.Partial n pst1 -> do

829+ assert (n <= Prelude.length (x:getList backBuf)) (return ())

830+ let src0 = Prelude.take n (x:getList backBuf)

831+ arr0 = A.fromListN n (Prelude.reverse src0)

832+ arr1 = Array contents next end

833+ src = arr0 <> arr1

834+ let !(Array cont1 start end1) = src

835+ fp1 = Tuple' end1 cont1

836+ gobuf SPEC s (List []) fp1 start pst1

837+ PR.Continue 0 pst1 ->

838+ gobuf SPEC s (List (x:getList backBuf)) fp next pst1

839+ PR.Continue n pst1 -> do

840+ assert (n <= Prelude.length (x:getList backBuf)) (return ())

841+ let (src0, buf1) = splitAt n (x:getList backBuf)

842+ arr0 = A.fromListN n (Prelude.reverse src0)

843+ arr1 = Array contents next end

844+ src = arr0 <> arr1

845+ let !(Array cont1 start end1) = src

846+ fp1 = Tuple' end1 cont1

847+ gobuf SPEC s (List buf1) fp1 start pst1

848+ PR.Done 0 b -> do

849+ let arr = Array contents next end

850+ return (b, D.cons arr (D.Stream step s))

851+ PR.Done n b -> do

852+ assert (n <= Prelude.length (x:getList backBuf)) (return ())

853+ let src0 = Prelude.take n (x:getList backBuf)

854+ -- XXX create the array in reverse instead

855+ arr0 = A.fromListN n (Prelude.reverse src0)

856+ arr1 = Array contents next end

857+ -- XXX Use StreamK to avoid adding arbitrary layers of

858+ -- constructors every time.

859+ str = D.cons arr0 (D.cons arr1 (D.Stream step s))

860+ return (b, str)

861+ PR.Error err -> throwM $ ParseError err

862+-}

863+

864+-- | Parse an array stream using the supplied 'Parser'. Returns the parse

865+-- result and the unconsumed stream. Throws 'ParseError' if the parse fails.

866+--

867+-- The following alternative to this function allows composing the parser using

868+-- the parser Monad:

869+--

870+-- >>> parseBreakStreamK p = StreamK.parseBreakChunks (ParserK.adaptC p)

871+--

872+-- We can compare perf and remove this one or define it in terms of that.

873+--

874+-- /Internal/

875+--

876+{-# INLINE_NORMAL parseBreakChunksK #-}

877+parseBreakChunksK ::

878+ forall m a b. (MonadIO m, Unbox a)

879+ => Parser a m b

880+ -> StreamK m (Array a)

881+ -> m (Either ParseError b, StreamK m (Array a))

882+-- parseBreakStreamK p = StreamK.parseBreakChunks (ParserK.adaptC p)

883+parseBreakChunksK (Parser pstep initial extract) stream = do

884+ res <- initial

885+ case res of

886+ IPartial s -> go s stream []

887+ IDone b -> return (Right b, stream)

888+ IError err -> return (Left (ParseError err), stream)

889+

890+ where

891+

892+ -- "backBuf" contains last few items in the stream that we may have to

893+ -- backtrack to.

894+ --

895+ -- XXX currently we are using a dumb list based approach for backtracking

896+ -- buffer. This can be replaced by a sliding/ring buffer using Data.Array.

897+ -- That will allow us more efficient random back and forth movement.

898+ go !pst st backBuf = do

899+ let stop = goStop pst backBuf -- (, K.nil) <$> extract pst

900+ single a = yieldk a StreamK.nil

901+ yieldk arr r = goArray pst backBuf r arr

902+ in StreamK.foldStream defState yieldk single stop st

903+

904+ -- Use strictness on "cur" to keep it unboxed

905+ goArray !pst backBuf st (Array _ cur end) | cur == end = go pst st backBuf

906+ goArray !pst backBuf st (Array contents cur end) = do

907+ x <- liftIO $ peekAt cur contents

908+ pRes <- pstep pst x

909+ let next = INDEX_NEXT(cur,a)

910+ case pRes of

911+ Parser.Partial 0 s ->

912+ goArray s [] st (Array contents next end)

913+ Parser.Partial n s -> do

914+ assert (n <= Prelude.length (x:backBuf)) (return ())

915+ let src0 = Prelude.take n (x:backBuf)

916+ arr0 = fromListN n (Prelude.reverse src0)

917+ arr1 = Array contents next end

918+ src = arr0 <> arr1

919+ goArray s [] st src

920+ Parser.Continue 0 s ->

921+ goArray s (x:backBuf) st (Array contents next end)

922+ Parser.Continue n s -> do

923+ assert (n <= Prelude.length (x:backBuf)) (return ())

924+ let (src0, buf1) = Prelude.splitAt n (x:backBuf)

925+ arr0 = fromListN n (Prelude.reverse src0)

926+ arr1 = Array contents next end

927+ src = arr0 <> arr1

928+ goArray s buf1 st src

929+ Parser.Done 0 b -> do

930+ let arr = Array contents next end

931+ return (Right b, StreamK.cons arr st)

932+ Parser.Done n b -> do

933+ assert (n <= Prelude.length (x:backBuf)) (return ())

934+ let src0 = Prelude.take n (x:backBuf)

935+ -- XXX Use fromListRevN once implemented

936+ -- arr0 = A.fromListRevN n src0

937+ arr0 = fromListN n (Prelude.reverse src0)

938+ arr1 = Array contents next end

939+ str = StreamK.cons arr0 (StreamK.cons arr1 st)

940+ return (Right b, str)

941+ Parser.Error err -> do

942+ let n = Prelude.length backBuf

943+ arr0 = fromListN n (Prelude.reverse backBuf)

944+ arr1 = Array contents cur end

945+ str = StreamK.cons arr0 (StreamK.cons arr1 stream)

946+ return (Left (ParseError err), str)

947+

948+ -- This is a simplified goArray

949+ goExtract !pst backBuf (Array _ cur end)

950+ | cur == end = goStop pst backBuf

951+ goExtract !pst backBuf (Array contents cur end) = do

952+ x <- liftIO $ peekAt cur contents

953+ pRes <- pstep pst x

954+ let next = INDEX_NEXT(cur,a)

955+ case pRes of

956+ Parser.Partial 0 s ->

957+ goExtract s [] (Array contents next end)

958+ Parser.Partial n s -> do

959+ assert (n <= Prelude.length (x:backBuf)) (return ())

960+ let src0 = Prelude.take n (x:backBuf)

961+ arr0 = fromListN n (Prelude.reverse src0)

962+ arr1 = Array contents next end

963+ src = arr0 <> arr1

964+ goExtract s [] src

965+ Parser.Continue 0 s ->

966+ goExtract s backBuf (Array contents next end)

967+ Parser.Continue n s -> do

968+ assert (n <= Prelude.length (x:backBuf)) (return ())

969+ let (src0, buf1) = Prelude.splitAt n (x:backBuf)

970+ arr0 = fromListN n (Prelude.reverse src0)

971+ arr1 = Array contents next end

972+ src = arr0 <> arr1

973+ goExtract s buf1 src

974+ Parser.Done 0 b -> do

975+ let arr = Array contents next end

976+ return (Right b, StreamK.fromPure arr)

977+ Parser.Done n b -> do

978+ assert (n <= Prelude.length backBuf) (return ())

979+ let src0 = Prelude.take n backBuf

980+ -- XXX Use fromListRevN once implemented

981+ -- arr0 = A.fromListRevN n src0

982+ arr0 = fromListN n (Prelude.reverse src0)

983+ arr1 = Array contents next end

984+ str = StreamK.cons arr0 (StreamK.fromPure arr1)

985+ return (Right b, str)

986+ Parser.Error err -> do

987+ let n = Prelude.length backBuf

988+ arr0 = fromListN n (Prelude.reverse backBuf)

989+ arr1 = Array contents cur end

990+ str = StreamK.cons arr0 (StreamK.cons arr1 stream)

991+ return (Left (ParseError err), str)

992+

993+ -- This is a simplified goExtract

994+ {-# INLINE goStop #-}

995+ goStop !pst backBuf = do

996+ pRes <- extract pst

997+ case pRes of

998+ Parser.Partial _ _ -> error "Bug: parseBreak: Partial in extract"

999+ Parser.Continue 0 s ->

1000+ goStop s backBuf

1001+ Parser.Continue n s -> do

1002+ assert (n <= Prelude.length backBuf) (return ())

1003+ let (src0, buf1) = Prelude.splitAt n backBuf

1004+ arr = fromListN n (Prelude.reverse src0)

1005+ goExtract s buf1 arr

1006+ Parser.Done 0 b ->

1007+ return (Right b, StreamK.nil)

1008+ Parser.Done n b -> do

1009+ assert (n <= Prelude.length backBuf) (return ())

1010+ let src0 = Prelude.take n backBuf

1011+ -- XXX Use fromListRevN once implemented

1012+ -- arr0 = A.fromListRevN n src0

1013+ arr0 = fromListN n (Prelude.reverse src0)

1014+ return (Right b, StreamK.fromPure arr0)

1015+ Parser.Error err -> do

1016+ let n = Prelude.length backBuf

1017+ arr0 = fromListN n (Prelude.reverse backBuf)

1018+ return (Left (ParseError err), StreamK.fromPure arr0)

core/src/Streamly/Internal/Data/Array/Generic.hs+22−6

@@ -11,8 +11,8 @@ module Streamly.Internal.Data.Array.Generic

1111

1212 -- * Construction

1313 , nil

14− , writeN

15− , write

14+ , createOf

15+ , create

1616 , writeWith

1717 , writeLastN

1818

@@ -44,6 +44,10 @@ module Streamly.Internal.Data.Array.Generic

4444 , getIndex

4545 , getSliceUnsafe

4646 , strip

47+

48+ -- * Deprecated

49+ , writeN

50+ , write

4751 )

4852where

4953

@@ -104,21 +108,33 @@ nil = unsafePerformIO $ unsafeFreeze <$> MArray.nil

104108-- Construction - Folds

105109-------------------------------------------------------------------------------

106110

107−{-# INLINE_NORMAL writeN #-}

111+{-# INLINE_NORMAL createOf #-}

112+createOf :: MonadIO m => Int -> Fold m a (Array a)

113+createOf = fmap unsafeFreeze <$> MArray.createOf

114+

115+-- XXX Deprecate in major

116+-- {-# DEPRECATED writeN "Please use createOf instead." #-}

117+{-# INLINE writeN #-}

108118writeN :: MonadIO m => Int -> Fold m a (Array a)

109−writeN = fmap unsafeFreeze <$> MArray.writeN

119+writeN = createOf

110120

111121{-# INLINE_NORMAL writeWith #-}

112122writeWith :: MonadIO m => Int -> Fold m a (Array a)

113−writeWith elemCount = unsafeFreeze <$> MArray.writeWith elemCount

123+writeWith elemCount = unsafeFreeze <$> MArray.createWith elemCount

114124

115125-- | Fold the whole input to a single array.

116126--

117127-- /Caution! Do not use this on infinite streams./

118128--

129+{-# INLINE create #-}

130+create :: MonadIO m => Fold m a (Array a)

131+create = fmap unsafeFreeze MArray.create

132+

133+-- XXX Deprecate in major

134+-- {-# DEPRECATED write "Please use create instead." #-}

119135{-# INLINE write #-}

120136write :: MonadIO m => Fold m a (Array a)

121−write = fmap unsafeFreeze MArray.write

137+write = create

122138

123139fromPureStream :: Stream Identity a -> Array a

124140fromPureStream x =

core/src/Streamly/Internal/Data/Stream/Chunked.hs → core/src/Streamly/Internal/Data/Array/Stream.hsrenamed+74−586

@@ -1,5 +1,6 @@

1+{-# OPTIONS_GHC -Wno-deprecations #-}

12-- |

2−-- Module : Streamly.Internal.Data.Stream.Chunked

3+-- Module : Streamly.Internal.Data.Array.Stream

45-- License : BSD3-3-Clause

56-- Maintainer : streamly@composewell.com

@@ -7,17 +8,25 @@

78--

89-- Combinators to efficiently manipulate streams of immutable arrays.

910--

10−module Streamly.Internal.Data.Stream.Chunked

11+-- We can either push these in the MutArray module with a "chunks" prefix or

12+-- keep this as a separate module and release it.

13+--

14+module Streamly.Internal.Data.Array.Stream

15+{-# DEPRECATED "Please use \"Streamly.Internal.Data.Array\" instead." #-}

1116 (

1217 -- * Creation

13− chunksOf

18+ Array.chunksOf

19+ , Array.pinnedChunksOf

20+ , Array.bufferChunks

1421

1522 -- * Flattening to elements

16− , concat

17− , concatRev

18− , interpose

19− , interposeSuffix

20− , intercalateSuffix

23+ , Array.concat

24+ , Array.flattenArrays

25+ , Array.concatRev

26+ , Array.flattenArraysRev

27+ , Array.interpose

28+ , Array.interposeSuffix

29+ , Array.intercalateSuffix

2130 , unlines

2231

2332 -- * Elimination

@@ -26,24 +35,27 @@ module Streamly.Internal.Data.Stream.Chunked

2635 -- can flatten the stream to byte stream and use that. But if we want the

2736 -- remaining stream to be a chunk stream then this could be handy. But it

2837 -- could also be implemented using parseBreak.

29− , foldBreak -- StreamK.foldBreakChunks

38+ , foldBreak

3039 , foldBreakD

31− -- The byte level parseBreak cannot work efficiently. Because the stream

32− -- will have to be a StreamK for backtracking, StreamK at byte level would

33− -- not be efficient.

34− , parseBreak -- StreamK.parseBreakChunks

40+ -- This is chunked parseBreak. A byte level parseBreak cannot work

41+ -- efficiently. Because the stream will have to be a StreamK for

42+ -- backtracking, StreamK at byte level would not be efficient.

43+ -- parseBreak p = K.parseBreakChunks (ParserK.adaptC p)

44+ , parseBreak

3545 -- , parseBreakD

3646 -- , foldManyChunks

3747 -- , parseManyChunks

48+ , K.parseBreakChunks

49+ , K.parseChunks

3850

3951 -- ** Array Folds

40− -- XXX Use Parser.Chunked instead, need only chunkedParseBreak,

52+ -- XXX Use parseBreakChunks/parseChunks instead

4153 -- foldBreak can be implemented using parseBreak. Use StreamK.

4254 , runArrayFold

4355 , runArrayFoldBreak

4456 -- , parseArr

45− , runArrayParserDBreak -- StreamK.chunkedParseBreak

46− , runArrayFoldMany -- StreamK.chunkedParseMany

57+ , runArrayParserDBreak -- StreamK.parseBreakChunks

58+ , runArrayFoldMany

4759

4860 , toArray

4961

@@ -67,6 +79,8 @@ where

6779import Data.Bifunctor (second)

6880import Control.Exception (assert)

6981import Control.Monad.IO.Class (MonadIO(..))

82+-- import Data.Bifunctor (first)

83+-- import Data.Either (fromRight)

7084import Data.Proxy (Proxy(..))

7185import Data.Word (Word8)

7286import Streamly.Internal.Data.Unbox (Unbox(..))

@@ -82,142 +96,32 @@ import Streamly.Internal.Data.Parser (ParseError(..))

8296import Streamly.Internal.Data.Stream (Stream)

8397import Streamly.Internal.Data.StreamK (StreamK, fromStream, toStream)

8498import Streamly.Internal.Data.SVar.Type (adaptState, defState)

85−import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))

8699

87−import qualified Streamly.Data.Fold as FL

88100import qualified Streamly.Internal.Data.Array as A

89101import qualified Streamly.Internal.Data.Array as Array

90−import qualified Streamly.Internal.Data.MutArray.Type as MA

91−import qualified Streamly.Internal.Data.Stream.MutChunked as AS

92−import qualified Streamly.Internal.Data.Fold.Type as FL (Fold(..), Step(..))

93102import qualified Streamly.Internal.Data.Parser as PR

94103import qualified Streamly.Internal.Data.Parser as PRD

95104 (Parser(..), Initial(..))

105+-- import qualified Streamly.Internal.Data.ParserK as ParserK

96106import qualified Streamly.Internal.Data.Stream as D

97107import qualified Streamly.Internal.Data.StreamK as K

98108

99109-- XXX Since these are immutable arrays MonadIO constraint can be removed from

100110-- most places.

101111

102−-------------------------------------------------------------------------------

103−-- Generation

104−-------------------------------------------------------------------------------

105−

106−-- | @chunksOf n stream@ groups the elements in the input stream into arrays of

107−-- @n@ elements each.

108−--

109−-- > chunksOf n = Stream.groupsOf n (Array.writeN n)

110−--

111−-- /Pre-release/

112−{-# INLINE chunksOf #-}

113−chunksOf :: (MonadIO m, Unbox a)

114− => Int -> Stream m a -> Stream m (Array a)

115−chunksOf = A.chunksOf

116−

117−-------------------------------------------------------------------------------

118−-- Append

119−-------------------------------------------------------------------------------

120−

121−-- XXX efficiently compare two streams of arrays. Two streams can have chunks

122−-- of different sizes, we can handle that in the stream comparison abstraction.

123−-- This could be useful e.g. to fast compare whether two files differ.

124−

125−-- | Convert a stream of arrays into a stream of their elements.

126−--

127−-- Same as the following:

128−--

129−-- > concat = Stream.unfoldMany Array.read

130−--

131−-- @since 0.7.0

132−{-# INLINE concat #-}

133−concat :: (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

134−-- concat m = fromStreamD $ A.flattenArrays (toStreamD m)

135−-- concat m = fromStreamD $ D.concatMap A.toStreamD (toStreamD m)

136−concat = D.unfoldMany A.reader

137−

138−-- | Convert a stream of arrays into a stream of their elements reversing the

139−-- contents of each array before flattening.

140−--

141−-- > concatRev = Stream.unfoldMany Array.readerRev

142−--

143−-- @since 0.7.0

144−{-# INLINE concatRev #-}

145−concatRev :: (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

146−-- concatRev m = fromStreamD $ A.flattenArraysRev (toStreamD m)

147−concatRev = D.unfoldMany A.readerRev

148−

149112-------------------------------------------------------------------------------

150113-- Intersperse and append

151114-------------------------------------------------------------------------------

152115

153−-- | Flatten a stream of arrays after inserting the given element between

154−-- arrays.

155−--

156−-- /Pre-release/

157−{-# INLINE interpose #-}

158−interpose :: (Monad m, Unbox a) => a -> Stream m (Array a) -> Stream m a

159−interpose x = D.interpose x A.reader

160−

161−{-# INLINE intercalateSuffix #-}

162−intercalateSuffix :: (Monad m, Unbox a)

163− => Array a -> Stream m (Array a) -> Stream m a

164−intercalateSuffix = D.intercalateSuffix A.reader

165−

166−-- | Flatten a stream of arrays appending the given element after each

167−-- array.

168−--

169−-- @since 0.7.0

170−{-# INLINE interposeSuffix #-}

171−interposeSuffix :: (Monad m, Unbox a)

172− => a -> Stream m (Array a) -> Stream m a

173−-- interposeSuffix x = fromStreamD . A.unlines x . toStreamD

174−interposeSuffix x = D.interposeSuffix x A.reader

175−

176−data FlattenState s =

177− OuterLoop s

178− | InnerLoop s !MA.MutableByteArray !Int !Int

179−

180−-- XXX This is a special case of interposeSuffix, can be removed.

181−-- XXX Remove monadIO constraint

182116{-# INLINE_NORMAL unlines #-}

183117unlines :: forall m a. (MonadIO m, Unbox a)

184118 => a -> D.Stream m (Array a) -> D.Stream m a

185−unlines sep (D.Stream step state) = D.Stream step' (OuterLoop state)

186− where

187− {-# INLINE_LATE step' #-}

188− step' gst (OuterLoop st) = do

189− r <- step (adaptState gst) st

190− return $ case r of

191− D.Yield Array{..} s ->

192− D.Skip (InnerLoop s arrContents arrStart arrEnd)

193− D.Skip s -> D.Skip (OuterLoop s)

194− D.Stop -> D.Stop

195−

196− step' _ (InnerLoop st _ p end) | p == end =

197− return $ D.Yield sep $ OuterLoop st

198−

199− step' _ (InnerLoop st contents p end) = do

200− x <- liftIO $ peekByteIndex p contents

201− return $ D.Yield x (InnerLoop st contents (INDEX_NEXT(p,a)) end)

119+unlines = Array.interposeSuffix

202120

203121-------------------------------------------------------------------------------

204122-- Compact

205123-------------------------------------------------------------------------------

206124

207−-- XXX These would not be needed once we implement compactLEFold, see

208−-- module Streamly.Internal.Data.Stream.MutChunked

209−--

210−-- XXX Note that this thaws immutable arrays for appending, that may be

211−-- problematic if multiple users do the same thing, however, immutable arrays

212−-- would usually have no capacity to append, therefore, a copy will be forced

213−-- anyway. Confirm this. We can forcefully trim the array capacity before thaw

214−-- to ensure this.

215−{-# INLINE_NORMAL packArraysChunksOf #-}

216−packArraysChunksOf :: (MonadIO m, Unbox a)

217− => Int -> D.Stream m (Array a) -> D.Stream m (Array a)

218−packArraysChunksOf n str =

219− D.map A.unsafeFreeze $ AS.packArraysChunksOf n $ D.map A.unsafeThaw str

220−

221125-- XXX instead of writing two different versions of this operation, we should

222126-- write it as a pipe.

223127--

@@ -225,8 +129,7 @@ packArraysChunksOf n str =

225129{-# INLINE_NORMAL lpackArraysChunksOf #-}

226130lpackArraysChunksOf :: (MonadIO m, Unbox a)

227131 => Int -> Fold m (Array a) () -> Fold m (Array a) ()

228−lpackArraysChunksOf n fld =

229− FL.lmap A.unsafeThaw $ AS.lpackArraysChunksOf n (FL.lmap A.unsafeFreeze fld)

132+lpackArraysChunksOf = Array.lCompactGE

230133

231134-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

232135-- maximum specified size in bytes.

@@ -235,68 +138,19 @@ lpackArraysChunksOf n fld =

235138{-# INLINE compact #-}

236139compact :: (MonadIO m, Unbox a)

237140 => Int -> Stream m (Array a) -> Stream m (Array a)

238−compact = packArraysChunksOf

141+compact = Array.compactLE

239142

240−-------------------------------------------------------------------------------

241−-- Split

242−-------------------------------------------------------------------------------

243−

244−data SplitState s arr

245− = Initial s

246− | Buffering s arr

247− | Splitting s arr

248− | Yielding arr (SplitState s arr)

249− | Finishing

250−

251−-- | Split a stream of arrays on a given separator byte, dropping the separator

252−-- and coalescing all the arrays between two separators into a single array.

143+-- | Given a stream of arrays, splice them all together to generate a single

144+-- array. The stream must be /finite/.

253145--

254146-- @since 0.7.0

255−{-# INLINE_NORMAL _splitOn #-}

256−_splitOn

257− :: MonadIO m

258− => Word8

259− -> D.Stream m (Array Word8)

260− -> D.Stream m (Array Word8)

261−_splitOn byte (D.Stream step state) = D.Stream step' (Initial state)

262−

263− where

264−

265− {-# INLINE_LATE step' #-}

266− step' gst (Initial st) = do

267− r <- step gst st

268− case r of

269− D.Yield arr s -> do

270− (arr1, marr2) <- A.breakOn byte arr

271− return $ case marr2 of

272− Nothing -> D.Skip (Buffering s arr1)

273− Just arr2 -> D.Skip (Yielding arr1 (Splitting s arr2))

274− D.Skip s -> return $ D.Skip (Initial s)

275− D.Stop -> return D.Stop

276−

277− step' gst (Buffering st buf) = do

278− r <- step gst st

279− case r of

280− D.Yield arr s -> do

281− (arr1, marr2) <- A.breakOn byte arr

282− buf' <- A.splice buf arr1

283− return $ case marr2 of

284− Nothing -> D.Skip (Buffering s buf')

285− Just x -> D.Skip (Yielding buf' (Splitting s x))

286− D.Skip s -> return $ D.Skip (Buffering s buf)

287− D.Stop -> return $

288− if A.byteLength buf == 0

289− then D.Stop

290− else D.Skip (Yielding buf Finishing)

291−

292− step' _ (Splitting st buf) = do

293− (arr1, marr2) <- A.breakOn byte buf

294− return $ case marr2 of

295− Nothing -> D.Skip $ Buffering st arr1

296− Just arr2 -> D.Skip $ Yielding arr1 (Splitting st arr2)

147+{-# INLINE toArray #-}

148+toArray :: (MonadIO m, Unbox a) => Stream m (Array a) -> m (Array a)

149+toArray = Array.fromChunks

297150

298− step' _ (Yielding arr next) = return $ D.Yield arr next

299− step' _ Finishing = return D.Stop

151+-------------------------------------------------------------------------------

152+-- Split

153+-------------------------------------------------------------------------------

300154

301155-- XXX Remove MonadIO constraint.

302156-- | Split a stream of arrays on a given separator byte, dropping the separator

@@ -309,7 +163,7 @@ splitOn

309163 => Word8

310164 -> Stream m (Array Word8)

311165 -> Stream m (Array Word8)

312−splitOn byte = D.splitInnerBy (A.breakOn byte) A.splice

166+splitOn = Array.compactOnByte

313167

314168{-# INLINE splitOnSuffix #-}

315169splitOnSuffix

@@ -317,8 +171,7 @@ splitOnSuffix

317171 => Word8

318172 -> Stream m (Array Word8)

319173 -> Stream m (Array Word8)

320−-- splitOn byte s = fromStreamD $ A.splitOn byte $ toStreamD s

321−splitOnSuffix byte = D.splitInnerBySuffix (A.breakOn byte) A.splice

174+splitOnSuffix = Array.compactOnByteSuffix

322175

323176-------------------------------------------------------------------------------

324177-- Elimination - Running folds

@@ -327,77 +180,24 @@ splitOnSuffix byte = D.splitInnerBySuffix (A.breakOn byte) A.splice

327180{-# INLINE_NORMAL foldBreakD #-}

328181foldBreakD :: forall m a b. (MonadIO m, Unbox a) =>

329182 Fold m a b -> D.Stream m (Array a) -> m (b, D.Stream m (Array a))

330−foldBreakD (FL.Fold fstep initial _ final) stream@(D.Stream step state) = do

331− res <- initial

332− case res of

333− FL.Partial fs -> go SPEC state fs

334− FL.Done fb -> return $! (fb, stream)

335−

336− where

337−

338− {-# INLINE go #-}

339− go !_ st !fs = do

340− r <- step defState st

341− case r of

342− D.Yield (Array contents start end) s ->

343− let fp = Tuple' end contents

344− in goArray SPEC s fp start fs

345− D.Skip s -> go SPEC s fs

346− D.Stop -> do

347− b <- final fs

348− return (b, D.nil)

349−

350− goArray !_ s (Tuple' end _) !cur !fs

351− | cur == end = do

352− go SPEC s fs

353− goArray !_ st fp@(Tuple' end contents) !cur !fs = do

354− x <- liftIO $ peekByteIndex cur contents

355− res <- fstep fs x

356− let next = INDEX_NEXT(cur,a)

357− case res of

358− FL.Done b -> do

359− let arr = Array contents next end

360− return $! (b, D.cons arr (D.Stream step st))

361− FL.Partial fs1 -> goArray SPEC st fp next fs1

362−

363−{-# INLINE_NORMAL foldBreakK #-}

364−foldBreakK :: forall m a b. (MonadIO m, Unbox a) =>

365− Fold m a b -> K.StreamK m (Array a) -> m (b, K.StreamK m (Array a))

366−foldBreakK (FL.Fold fstep initial _ final) stream = do

367− res <- initial

368− case res of

369− FL.Partial fs -> go fs stream

370− FL.Done fb -> return (fb, stream)

371−

372− where

373−

374− {-# INLINE go #-}

375− go !fs st = do

376− let stop = (, K.nil) <$> final fs

377− single a = yieldk a K.nil

378− yieldk (Array contents start end) r =

379− let fp = Tuple' end contents

380− in goArray fs r fp start

381− in K.foldStream defState yieldk single stop st

382−

383− goArray !fs st (Tuple' end _) !cur

384− | cur == end = do

385− go fs st

386− goArray !fs st fp@(Tuple' end contents) !cur = do

387− x <- liftIO $ peekByteIndex cur contents

388− res <- fstep fs x

389− let next = INDEX_NEXT(cur,a)

390− case res of

391− FL.Done b -> do

392− let arr = Array contents next end

393− return $! (b, K.cons arr st)

394− FL.Partial fs1 -> goArray fs1 st fp next

183+foldBreakD = Array.foldBreakChunks

395184

396185-- | Fold an array stream using the supplied 'Fold'. Returns the fold result

397186-- and the unconsumed stream.

398187--

399188-- > foldBreak f = runArrayFoldBreak (ChunkFold.fromFold f)

400189--

190+-- Instead of using this we can adapt the fold to ParserK and use

191+-- parseBreakChunks instead. ParserK allows composing using Monad as well.

192+--

193+-- @

194+-- foldBreak f s =

195+-- fmap (first (fromRight undefined))

196+-- $ K.parseBreakChunks (ParserK.adaptC (PR.fromFold f)) s

197+-- @

198+--

199+-- We can compare perf and remove this one or define it in terms of that.

200+--

401201-- /Internal/

402202--

403203{-# INLINE_NORMAL foldBreak #-}

@@ -406,14 +206,20 @@ foldBreak ::

406206 => Fold m a b

407207 -> StreamK m (A.Array a)

408208 -> m (b, StreamK m (A.Array a))

209+foldBreak = Array.foldBreakChunksK

210+--

409211-- foldBreak f s = fmap fromStreamD <$> foldBreakD f (toStreamD s)

410−foldBreak = foldBreakK

212+--

213+-- foldBreak f s =

214+-- fmap (first (fromRight undefined))

215+-- $ K.parseBreakChunks (ParserK.adaptC (PR.fromFold f)) s

216+--

411217-- If foldBreak performs better than runArrayFoldBreak we can use a rewrite

412218-- rule to rewrite runArrayFoldBreak to fold.

413219-- foldBreak f = runArrayFoldBreak (ChunkFold.fromFold f)

414220

415221-------------------------------------------------------------------------------

416−-- Fold to a single Array

222+-- Elimination - running element parsers

417223-------------------------------------------------------------------------------

418224

419225-- When we have to take an array partially, take the last part of the array.

@@ -459,341 +265,22 @@ splitAtArrayListRev n ls

459265 arr1 = Array contents end1 end

460266 in ([arr1], arr2:xs)

461267

462−-------------------------------------------------------------------------------

463−-- Fold to a single Array

464−-------------------------------------------------------------------------------

465−

466−{-

467−

468−-- XXX Both of these implementations of splicing seem to perform equally well.

469−-- We need to perform benchmarks over a range of sizes though.

470−

471−-- CAUTION! length must more than equal to lengths of all the arrays in the

472−-- stream.

473−{-# INLINE spliceArraysLenUnsafe #-}

474−spliceArraysLenUnsafe :: (MonadIO m, Unbox a)

475− => Int -> Stream m (MutArray a) -> m (MutArray a)

476−spliceArraysLenUnsafe len buffered = do

477− -- XXX The new array's pinned state should depend on the first element

478− -- of the stream. We should uncons the stream and do the required.

479− arr <- undefined

480− D.foldlM' MA.spliceUnsafe (return arr) buffered

481−

482−{-# INLINE _spliceArrays #-}

483−_spliceArrays :: (MonadIO m, Unbox a)

484− => Stream m (Array a) -> m (Array a)

485−_spliceArrays s = do

486− buffered <- D.foldr K.cons K.nil s

487− len <- K.fold FL.sum (fmap Array.length buffered)

488− -- XXX The new array's pinned state should depend on the first element

489− -- of the stream. We should uncons the stream and do the required.

490− arr <- undefined

491− final <- D.foldlM' writeArr (return arr) (toStream buffered)

492− return $ A.unsafeFreeze final

493−

494− where

495−

496− writeArr dst arr = MA.spliceUnsafe dst (A.unsafeThaw arr)

497−

498−{-# INLINE _spliceArraysBuffered #-}

499−_spliceArraysBuffered :: (MonadIO m, Unbox a)

500− => Stream m (Array a) -> m (Array a)

501−_spliceArraysBuffered s = do

502− buffered <- D.foldr K.cons K.nil s

503− len <- K.fold FL.sum (fmap Array.length buffered)

504− A.unsafeFreeze <$>

505− spliceArraysLenUnsafe len (fmap A.unsafeThaw (toStream buffered))

506−

507−-}

508−

509−{-# INLINE spliceArraysRealloced #-}

510−spliceArraysRealloced :: forall m a. (MonadIO m, Unbox a)

511− => Stream m (Array a) -> m (Array a)

512−spliceArraysRealloced s = do

513− res <- D.uncons s

514− case res of

515− Just (a, strm) -> do

516− arr <-

517− D.foldlM'

518− MA.spliceExp

519− (pure (A.unsafeThaw a))

520− (fmap A.unsafeThaw strm)

521− liftIO $ A.unsafeFreeze <$> MA.rightSize arr

522− Nothing -> pure A.nil

523−

524−-- XXX This should just be "fold A.write"

525−--

526−-- | Given a stream of arrays, splice them all together to generate a single

527−-- array. The stream must be /finite/.

528−--

529−-- @since 0.7.0

530−{-# INLINE toArray #-}

531−toArray :: (MonadIO m, Unbox a) => Stream m (Array a) -> m (Array a)

532−toArray = spliceArraysRealloced

533−-- spliceArrays = _spliceArraysBuffered

534−

535−-- exponentially increasing sizes of the chunks upto the max limit.

536−-- XXX this will be easier to implement with parsers/terminating folds

537−-- With this we should be able to reduce the number of chunks/allocations.

538−-- The reallocation/copy based toArray can also be implemented using this.

539−--

540−{-

541−{-# INLINE toArraysInRange #-}

542−toArraysInRange :: (MonadIO m, Unbox a)

543− => Int -> Int -> Fold m (Array a) b -> Fold m a b

544−toArraysInRange low high (Fold step initial extract) =

545−-}

546−

547−{-

548−-- | Fold the input to a pure buffered stream (List) of arrays.

549−{-# INLINE _toArraysOf #-}

550−_toArraysOf :: (MonadIO m, Unbox a)

551− => Int -> Fold m a (Stream Identity (Array a))

552−_toArraysOf n = FL.groupsOf n (A.writeNF n) FL.toStream

553−-}

554−

555−-------------------------------------------------------------------------------

556−-- Elimination - running element parsers

557−-------------------------------------------------------------------------------

558−

559268-- GHC parser does not accept {-# ANN type [] NoSpecConstr #-}, so we need

560269-- to make a newtype.

561270{-# ANN type List NoSpecConstr #-}

562271newtype List a = List {getList :: [a]}

563272

564−{-

565−-- This can be generalized to any type provided it can be unfolded to a stream

566−-- and it can be combined using a semigroup operation.

567−--

568−-- XXX This should be written using CPS (as parseK) if we want it to scale wrt

569−-- to the number of times it can be called on the same stream.

570−{-# INLINE_NORMAL parseBreakD #-}

571−parseBreakD ::

572− forall m a b. (MonadIO m, MonadThrow m, Unbox a)

573− => PRD.Parser a m b

574− -> D.Stream m (Array.Array a)

575− -> m (b, D.Stream m (Array.Array a))

576−parseBreakD

577− (PRD.Parser pstep initial extract) stream@(D.Stream step state) = do

578−

579− res <- initial

580− case res of

581− PRD.IPartial s -> go SPEC state (List []) s

582− PRD.IDone b -> return (b, stream)

583− PRD.IError err -> throwM $ ParseError err

584−

585− where

586−

587− -- "backBuf" contains last few items in the stream that we may have to

588− -- backtrack to.

589− --

590− -- XXX currently we are using a dumb list based approach for backtracking

591− -- buffer. This can be replaced by a sliding/ring buffer using Data.Array.

592− -- That will allow us more efficient random back and forth movement.

593− go !_ st backBuf !pst = do

594− r <- step defState st

595− case r of

596− D.Yield (Array contents start end) s ->

597− gobuf SPEC s backBuf

598− (Tuple' end contents) start pst

599− D.Skip s -> go SPEC s backBuf pst

600− D.Stop -> do

601− b <- extract pst

602− return (b, D.nil)

603−

604− -- Use strictness on "cur" to keep it unboxed

605− gobuf !_ s backBuf (Tuple' end _) !cur !pst

606− | cur == end = do

607− go SPEC s backBuf pst

608− gobuf !_ s backBuf fp@(Tuple' end contents) !cur !pst = do

609− x <- liftIO $ peekByteIndex contents cur

610− pRes <- pstep pst x

611− let next = INDEX_NEXT(cur,a)

612− case pRes of

613− PR.Partial 0 pst1 ->

614− gobuf SPEC s (List []) fp next pst1

615− PR.Partial n pst1 -> do

616− assert (n <= Prelude.length (x:getList backBuf)) (return ())

617− let src0 = Prelude.take n (x:getList backBuf)

618− arr0 = A.fromListN n (Prelude.reverse src0)

619− arr1 = Array contents next end

620− src = arr0 <> arr1

621− let !(Array cont1 start end1) = src

622− fp1 = Tuple' end1 cont1

623− gobuf SPEC s (List []) fp1 start pst1

624− PR.Continue 0 pst1 ->

625− gobuf SPEC s (List (x:getList backBuf)) fp next pst1

626− PR.Continue n pst1 -> do

627− assert (n <= Prelude.length (x:getList backBuf)) (return ())

628− let (src0, buf1) = splitAt n (x:getList backBuf)

629− arr0 = A.fromListN n (Prelude.reverse src0)

630− arr1 = Array contents next end

631− src = arr0 <> arr1

632− let !(Array cont1 start end1) = src

633− fp1 = Tuple' end1 cont1

634− gobuf SPEC s (List buf1) fp1 start pst1

635− PR.Done 0 b -> do

636− let arr = Array contents next end

637− return (b, D.cons arr (D.Stream step s))

638− PR.Done n b -> do

639− assert (n <= Prelude.length (x:getList backBuf)) (return ())

640− let src0 = Prelude.take n (x:getList backBuf)

641− -- XXX create the array in reverse instead

642− arr0 = A.fromListN n (Prelude.reverse src0)

643− arr1 = Array contents next end

644− -- XXX Use StreamK to avoid adding arbitrary layers of

645− -- constructors every time.

646− str = D.cons arr0 (D.cons arr1 (D.Stream step s))

647− return (b, str)

648− PR.Error err -> throwM $ ParseError err

649−-}

650−

651−{-# INLINE_NORMAL parseBreakK #-}

652−parseBreakK ::

653− forall m a b. (MonadIO m, Unbox a)

654− => PRD.Parser a m b

655− -> K.StreamK m (Array.Array a)

656− -> m (Either ParseError b, K.StreamK m (Array.Array a))

657−parseBreakK (PRD.Parser pstep initial extract) stream = do

658− res <- initial

659− case res of

660− PRD.IPartial s -> go s stream []

661− PRD.IDone b -> return (Right b, stream)

662− PRD.IError err -> return (Left (ParseError err), stream)

663−

664− where

665−

666− -- "backBuf" contains last few items in the stream that we may have to

667− -- backtrack to.

668− --

669− -- XXX currently we are using a dumb list based approach for backtracking

670− -- buffer. This can be replaced by a sliding/ring buffer using Data.Array.

671− -- That will allow us more efficient random back and forth movement.

672− go !pst st backBuf = do

673− let stop = goStop pst backBuf -- (, K.nil) <$> extract pst

674− single a = yieldk a K.nil

675− yieldk arr r = goArray pst backBuf r arr

676− in K.foldStream defState yieldk single stop st

677−

678− -- Use strictness on "cur" to keep it unboxed

679− goArray !pst backBuf st (Array _ cur end) | cur == end = go pst st backBuf

680− goArray !pst backBuf st (Array contents cur end) = do

681− x <- liftIO $ peekByteIndex cur contents

682− pRes <- pstep pst x

683− let next = INDEX_NEXT(cur,a)

684− case pRes of

685− PR.Partial 0 s ->

686− goArray s [] st (Array contents next end)

687− PR.Partial n s -> do

688− assert (n <= Prelude.length (x:backBuf)) (return ())

689− let src0 = Prelude.take n (x:backBuf)

690− arr0 = A.fromListN n (Prelude.reverse src0)

691− arr1 = Array contents next end

692− src = arr0 <> arr1

693− goArray s [] st src

694− PR.Continue 0 s ->

695− goArray s (x:backBuf) st (Array contents next end)

696− PR.Continue n s -> do

697− assert (n <= Prelude.length (x:backBuf)) (return ())

698− let (src0, buf1) = splitAt n (x:backBuf)

699− arr0 = A.fromListN n (Prelude.reverse src0)

700− arr1 = Array contents next end

701− src = arr0 <> arr1

702− goArray s buf1 st src

703− PR.Done 0 b -> do

704− let arr = Array contents next end

705− return (Right b, K.cons arr st)

706− PR.Done n b -> do

707− assert (n <= Prelude.length (x:backBuf)) (return ())

708− let src0 = Prelude.take n (x:backBuf)

709− -- XXX Use fromListRevN once implemented

710− -- arr0 = A.fromListRevN n src0

711− arr0 = A.fromListN n (Prelude.reverse src0)

712− arr1 = Array contents next end

713− str = K.cons arr0 (K.cons arr1 st)

714− return (Right b, str)

715− PR.Error err -> do

716− let n = Prelude.length backBuf

717− arr0 = A.fromListN n (Prelude.reverse backBuf)

718− arr1 = Array contents cur end

719− str = K.cons arr0 (K.cons arr1 stream)

720− return (Left (ParseError err), str)

721−

722− -- This is a simplified goArray

723− goExtract !pst backBuf (Array _ cur end)

724− | cur == end = goStop pst backBuf

725− goExtract !pst backBuf (Array contents cur end) = do

726− x <- liftIO $ peekByteIndex cur contents

727− pRes <- pstep pst x

728− let next = INDEX_NEXT(cur,a)

729− case pRes of

730− PR.Partial 0 s ->

731− goExtract s [] (Array contents next end)

732− PR.Partial n s -> do

733− assert (n <= Prelude.length (x:backBuf)) (return ())

734− let src0 = Prelude.take n (x:backBuf)

735− arr0 = A.fromListN n (Prelude.reverse src0)

736− arr1 = Array contents next end

737− src = arr0 <> arr1

738− goExtract s [] src

739− PR.Continue 0 s ->

740− goExtract s backBuf (Array contents next end)

741− PR.Continue n s -> do

742− assert (n <= Prelude.length (x:backBuf)) (return ())

743− let (src0, buf1) = splitAt n (x:backBuf)

744− arr0 = A.fromListN n (Prelude.reverse src0)

745− arr1 = Array contents next end

746− src = arr0 <> arr1

747− goExtract s buf1 src

748− PR.Done 0 b -> do

749− let arr = Array contents next end

750− return (Right b, K.fromPure arr)

751− PR.Done n b -> do

752− assert (n <= Prelude.length backBuf) (return ())

753− let src0 = Prelude.take n backBuf

754− -- XXX Use fromListRevN once implemented

755− -- arr0 = A.fromListRevN n src0

756− arr0 = A.fromListN n (Prelude.reverse src0)

757− arr1 = Array contents next end

758− str = K.cons arr0 (K.fromPure arr1)

759− return (Right b, str)

760− PR.Error err -> do

761− let n = Prelude.length backBuf

762− arr0 = A.fromListN n (Prelude.reverse backBuf)

763− arr1 = Array contents cur end

764− str = K.cons arr0 (K.cons arr1 stream)

765− return (Left (ParseError err), str)

766−

767− -- This is a simplified goExtract

768− {-# INLINE goStop #-}

769− goStop !pst backBuf = do

770− pRes <- extract pst

771− case pRes of

772− PR.Partial _ _ -> error "Bug: parseBreak: Partial in extract"

773− PR.Continue 0 s ->

774− goStop s backBuf

775− PR.Continue n s -> do

776− assert (n <= Prelude.length backBuf) (return ())

777− let (src0, buf1) = splitAt n backBuf

778− arr = A.fromListN n (Prelude.reverse src0)

779− goExtract s buf1 arr

780− PR.Done 0 b ->

781− return (Right b, K.nil)

782− PR.Done n b -> do

783− assert (n <= Prelude.length backBuf) (return ())

784− let src0 = Prelude.take n backBuf

785− -- XXX Use fromListRevN once implemented

786− -- arr0 = A.fromListRevN n src0

787− arr0 = A.fromListN n (Prelude.reverse src0)

788− return (Right b, K.fromPure arr0)

789− PR.Error err -> do

790− let n = Prelude.length backBuf

791− arr0 = A.fromListN n (Prelude.reverse backBuf)

792− return (Left (ParseError err), K.fromPure arr0)

793−

794273-- | Parse an array stream using the supplied 'Parser'. Returns the parse

795274-- result and the unconsumed stream. Throws 'ParseError' if the parse fails.

796275--

276+-- >> parseBreak p = K.parseBreakChunks (ParserK.adaptC p)

277+--

278+-- This is redundant and we can just use parseBreakChunks, as ParserK can be

279+-- composed using Monad. The only advantage of this is that we do not need to

280+-- adapt.

281+--

282+-- We can compare perf and remove this one or define it in terms of that.

283+--

797284-- /Internal/

798285--

799286{-# INLINE_NORMAL parseBreak #-}

@@ -806,7 +293,8 @@ parseBreak ::

806293parseBreak p s =

807294 fmap fromStreamD <$> parseBreakD (PRD.fromParserK p) (toStreamD s)

808295-}

809−parseBreak = parseBreakK

296+parseBreak = Array.parseBreakChunksK

297+-- parseBreak p = K.parseBreakChunks (ParserK.adaptC p)

810298

811299-------------------------------------------------------------------------------

812300-- Elimination - Running Array Folds and parsers

core/src/Streamly/Internal/Data/Array/Type.hs+515−108

@@ -1,8 +1,11 @@

11{-# LANGUAGE CPP #-}

2+{-# LANGUAGE TypeFamilies #-}

3+-- Must come after TypeFamilies, otherwise it is re-enabled.

4+-- MonoLocalBinds enabled by TypeFamilies causes perf regressions in general.

5+{-# LANGUAGE NoMonoLocalBinds #-}

26-- |

37-- Module : Streamly.Internal.Data.Array.Type

5−--

69-- License : BSD3-3-Clause

710-- Maintainer : streamly@composewell.com

811-- Stability : experimental

@@ -10,86 +13,146 @@

1013--

1114-- See notes in "Streamly.Internal.Data.MutArray.Type"

1215--

16+

1317module Streamly.Internal.Data.Array.Type

1418 (

19+ -- ** Type

1520 -- $arrayNotes

1621 Array (..)

17− , asPtrUnsafe

18− , nil

1922

20− -- * Freezing and Thawing

23+ -- ** Conversion

24+ -- *** Mutable and Immutable

2125 , unsafeFreeze

2226 , unsafeFreezeWithShrink

2327 , unsafeThaw

2428

25− -- * Pinning and Unpinning

29+ -- *** Pinned and Unpinned

2630 , pin

2731 , unpin

2832 , isPinned

2933

30− -- * Construction

31− , splice

34+ -- *** Casting

35+ , unsafePinnedAsPtr

3236

33− , fromList

34− , pinnedFromList

37+ -- ** Construction

38+ , empty

39+

40+ -- *** Cloning

41+ -- XXX Why would we clone an immutable array?

42+ , clone

43+ , pinnedClone

44+

45+ -- *** Slicing

46+ -- | Get a subarray without copying

47+ , splitAt

48+ , breakOn -- XXX requires MonadIO

49+

50+ -- *** Stream Folds

51+ , unsafeMakePure

52+ , createOf

53+ , pinnedCreateOf

54+ , unsafeCreateOf

55+ , unsafePinnedCreateOf

56+ , create

57+ , pinnedCreate

58+ , createWith

59+

60+ -- *** From containers

3561 , fromListN

3662 , pinnedFromListN

37− , fromListRev

63+ , fromList

64+ , pinnedFromList

3865 , fromListRevN

39− , fromStreamDN

40− , fromStreamD

66+ , fromListRev

67+ , fromStreamN

68+ , fromStream

69+ , fromPureStreamN

4170 , fromPureStream

4271 , fromByteStr#

72+ , fromByteStr

73+ , fromPtrN

74+ , fromChunks

75+ , fromChunksK

4376

44− -- * Split

45− , breakOn

77+ -- ** Reading

4678

47− -- ** Cloning arrays

48− , clone

49− , pinnedClone

79+ -- *** Indexing

80+ , unsafeIndexIO -- XXX unsafeGetIndexIO

81+ , getIndexUnsafe -- XXX unsafeGetIndex

5082

51− -- * Elimination

52− , unsafeIndexIO

53− , getIndexUnsafe

54− , byteLength

83+ -- *** To Streams

84+ , read

85+ , readRev

86+ , toStreamK

87+ , toStreamKRev

88+

89+ -- *** To Containers

90+ , toList

91+

92+ -- *** Unfolds

93+ , producer -- experimental

94+ , readerUnsafe

95+ , reader

96+ , readerRev

97+

98+ -- *** Size

5599 , length

100+ , byteLength

56101

102+ -- ** Folding

57103 , foldl'

58104 , foldr

59− , splitAt

105+ , byteCmp

106+ , byteEq

107+

108+ -- ** Appending

109+ , splice -- XXX requires MonadIO

110+

111+ -- ** Streams of arrays

112+ -- *** Chunk

113+ -- | Group a stream into arrays.

114+ , chunksOf

115+ , pinnedChunksOf

116+ , buildChunks

60117

118+ -- *** Split

119+ -- | Split an array into slices.

120+

121+ -- *** Concat

122+ -- | Append the arrays in a stream to form a stream of elements.

123+ , concat

124+ , concatRev

125+

126+ -- *** Compact

127+ -- | Append the arrays in a stream to form a stream of larger arrays.

128+ , fCompactGE

129+ , fPinnedCompactGE

130+ , lCompactGE

131+ , lPinnedCompactGE

132+ , compactGE

133+

134+ -- ** Deprecated

135+ , asPtrUnsafe

136+ , unsafeIndex

137+ , bufferChunks

138+ , flattenArrays

139+ , flattenArraysRev

140+ , fromArrayStreamK

141+ , fromStreamDN

142+ , fromStreamD

61143 , toStreamD

62144 , toStreamDRev

63− , toStreamK

64− , toStreamKRev

65145 , toStream

66146 , toStreamRev

67− , read

68− , readRev

69− , readerRev

70− , toList

71−

72− -- * Folds

147+ , nil

73148 , writeWith

74149 , writeN

75150 , pinnedWriteN

76151 , writeNUnsafe

77152 , pinnedWriteNUnsafe

78− , MA.ArrayUnsafe (..)

79153 , pinnedWriteNAligned

80154 , write

81155 , pinnedWrite

82− , unsafeMakePure

83−

84− -- * Streams of arrays

85− , chunksOf

86− , pinnedChunksOf

87− , bufferChunks

88− , flattenArrays

89− , flattenArraysRev

90−

91− -- * Deprecated

92− , unsafeIndex

93156 )

94157where

95158

@@ -97,31 +160,36 @@ where

97160#include "inline.hs"

98161

99162import Control.Exception (assert)

100−import Control.Monad (replicateM)

163+import Control.Monad (replicateM, when)

101164import Control.Monad.IO.Class (MonadIO(..))

102165import Data.Functor.Identity (Identity(..))

166+import Data.Int (Int8, Int16, Int32, Int64)

103167import Data.Proxy (Proxy(..))

104−import Data.Word (Word8)

168+import Data.Word (Word8, Word16, Word32, Word64)

105169import GHC.Base (build)

106170import GHC.Exts (IsList, IsString(..), Addr#)

107171

108172import GHC.IO (unsafePerformIO)

109173import GHC.Ptr (Ptr(..))

110−import Streamly.Internal.Data.MutArray.Type (MutArray(..), MutableByteArray)

174+import Streamly.Internal.Data.Producer.Type (Producer(..))

175+import Streamly.Internal.Data.MutArray.Type (MutArray(..))

176+import Streamly.Internal.Data.MutByteArray.Type (MutByteArray)

111177import Streamly.Internal.Data.Fold.Type (Fold(..))

112178import Streamly.Internal.Data.Stream.Type (Stream)

179+import Streamly.Internal.Data.StreamK.Type (StreamK)

113180import Streamly.Internal.Data.Unbox (Unbox(..))

114181import Streamly.Internal.Data.Unfold.Type (Unfold(..))

115182import Text.Read (readPrec)

116183

117−import Prelude hiding (Foldable(..), read, unlines, splitAt)

184+import Prelude hiding (Foldable(..), concat, read, unlines, splitAt)

118185

119186import qualified GHC.Exts as Exts

187+import qualified Streamly.Internal.Data.Fold.Type as Fold

120188import qualified Streamly.Internal.Data.MutArray.Type as MA

121189import qualified Streamly.Internal.Data.Stream.Type as D

122−import qualified Streamly.Internal.Data.Stream.Generate as D

123190import qualified Streamly.Internal.Data.StreamK.Type as K

124−import qualified Streamly.Internal.Data.Unbox as Unboxed

191+import qualified Streamly.Internal.Data.MutByteArray.Type as Unboxed

192+import qualified Streamly.Internal.Data.Producer as Producer

125193import qualified Streamly.Internal.Data.Unfold.Type as Unfold

126194import qualified Text.ParserCombinators.ReadPrec as ReadPrec

127195

@@ -164,7 +232,7 @@ data Array a =

164232#endif

165233 -- All offsets are in terms of bytes from the start of arraycontents

166234 Array

167− { arrContents :: {-# UNPACK #-} !MutableByteArray

235+ { arrContents :: {-# UNPACK #-} !MutByteArray

168236 , arrStart :: {-# UNPACK #-} !Int -- offset

169237 , arrEnd :: {-# UNPACK #-} !Int -- offset + len

170238 }

@@ -175,15 +243,20 @@ data Array a =

175243

176244-- | Use an @Array a@ as @Ptr a@.

177245--

178−-- See 'MA.asPtrUnsafe' in the Mutable array module for more details.

246+-- See 'MA.unsafePinnedAsPtr' in the Mutable array module for more details.

179247--

180248-- /Unsafe/

181249--

182250-- /Pre-release/

183251--

252+{-# INLINE unsafePinnedAsPtr #-}

253+unsafePinnedAsPtr :: MonadIO m => Array a -> (Ptr a -> m b) -> m b

254+unsafePinnedAsPtr arr = MA.unsafePinnedAsPtr (unsafeThaw arr)

255+

256+{-# DEPRECATED asPtrUnsafe "Please use unsafePinnedAsPtr instead." #-}

184257{-# INLINE asPtrUnsafe #-}

185258asPtrUnsafe :: MonadIO m => Array a -> (Ptr a -> m b) -> m b

186−asPtrUnsafe arr = MA.asPtrUnsafe (unsafeThaw arr)

259+asPtrUnsafe = unsafePinnedAsPtr

187260

188261-------------------------------------------------------------------------------

189262-- Freezing and Thawing

@@ -230,18 +303,19 @@ unsafeThaw (Array ac as ae) = MutArray ac as ae ae

230303-- Pinning & Unpinning

231304-------------------------------------------------------------------------------

232305

233−-- | Copy the 'Array' to pinned memory if unpinned, else do nothing. The

234−-- overhead is a copy if the input array is unpinned.

306+-- | Return a copy of the 'Array' in pinned memory if unpinned, else return the

307+-- original array.

235308{-# INLINE pin #-}

236309pin :: Array a -> IO (Array a)

237310pin = fmap unsafeFreeze . MA.pin . unsafeThaw

238311

239−-- | Copy the 'Array' to unpinned memory if pinned, else do nothing. The

240−-- overhead is a copy if the input array is pinned.

312+-- | Return a copy of the 'Array' in unpinned memory if pinned, else return the

313+-- original array.

241314{-# INLINE unpin #-}

242315unpin :: Array a -> IO (Array a)

243316unpin = fmap unsafeFreeze . MA.unpin . unsafeThaw

244317

318+-- | Return 'True' if the array is allocated in pinned memory.

245319{-# INLINE isPinned #-}

246320isPinned :: Array a -> Bool

247321isPinned = MA.isPinned . unsafeThaw

@@ -250,11 +324,14 @@ isPinned = MA.isPinned . unsafeThaw

250324-- Construction

251325-------------------------------------------------------------------------------

252326

253−-- Splice two immutable arrays creating a new array.

327+-- | Copy two immutable arrays into a new array. If you want to splice more

328+-- than two arrays then this operation would be highly inefficient because it

329+-- would make a copy on every splice operation, instead use the

330+-- 'fromChunksK' operation to combine n immutable arrays.

254331{-# INLINE splice #-}

255−splice :: (MonadIO m, Unbox a) => Array a -> Array a -> m (Array a)

332+splice :: MonadIO m => Array a -> Array a -> m (Array a)

256333splice arr1 arr2 =

257− unsafeFreeze <$> MA.splice (unsafeThaw arr1) (unsafeThaw arr2)

334+ unsafeFreeze <$> MA.spliceCopy (unsafeThaw arr1) (unsafeThaw arr2)

258335

259336-- | Create an 'Array' from the first N elements of a list. The array is

260337-- allocated to size N, if the list terminates before N elements then the

@@ -298,36 +375,83 @@ pinnedFromList xs = unsafePerformIO $ unsafeFreeze <$> MA.pinnedFromList xs

298375fromListRev :: Unbox a => [a] -> Array a

299376fromListRev xs = unsafePerformIO $ unsafeFreeze <$> MA.fromListRev xs

300377

301−{-# INLINE_NORMAL fromStreamDN #-}

378+-- | Create an 'Array' from the first N elements of a stream. The array is

379+-- allocated to size N, if the stream terminates before N elements then the

380+-- array may hold less than N elements.

381+--

382+-- >>> fromStreamN n = Stream.fold (Array.writeN n)

383+--

384+-- /Pre-release/

385+{-# INLINE_NORMAL fromStreamN #-}

386+fromStreamN :: (MonadIO m, Unbox a) => Int -> Stream m a -> m (Array a)

387+fromStreamN n m = do

388+ when (n < 0) $ error "writeN: negative write count specified"

389+ unsafeFreeze <$> MA.fromStreamN n m

390+-- fromStreamN n = D.fold (writeN n)

391+

392+{-# DEPRECATED fromStreamDN "Please use fromStreamN instead." #-}

302393fromStreamDN :: forall m a. (MonadIO m, Unbox a)

303394 => Int -> D.Stream m a -> m (Array a)

304−fromStreamDN limit str = unsafeFreeze <$> MA.fromStreamDN limit str

395+fromStreamDN = fromStreamN

305396

397+-- | Create an 'Array' from a stream. This is useful when we want to create a

398+-- single array from a stream of unknown size. 'writeN' is at least twice

399+-- as efficient when the size is already known.

400+--

401+-- >>> fromStream = Stream.fold Array.write

402+--

403+-- Note that if the input stream is too large memory allocation for the array

404+-- may fail. When the stream size is not known, `chunksOf` followed by

405+-- processing of indvidual arrays in the resulting stream should be preferred.

406+--

407+-- /Pre-release/

306408{-# INLINE_NORMAL fromStreamD #-}

409+fromStream :: (MonadIO m, Unbox a) => Stream m a -> m (Array a)

410+fromStream = D.fold write

411+-- fromStreamD str = unsafeFreeze <$> MA.fromStream str

412+

413+{-# DEPRECATED fromStreamD "Please use fromStream instead." #-}

307414fromStreamD :: forall m a. (MonadIO m, Unbox a)

308415 => D.Stream m a -> m (Array a)

309−fromStreamD str = unsafeFreeze <$> MA.fromStreamD str

416+fromStreamD = fromStream

310417

311418-------------------------------------------------------------------------------

312419-- Cloning

313420-------------------------------------------------------------------------------

314421

315422{-# INLINE clone #-}

316−clone :: MonadIO m => Array a -> m (Array a)

423+clone ::

424+ ( MonadIO m

425+#ifdef DEVBUILD

426+ , Unbox a

427+#endif

428+ )

429+ => Array a -> m (Array a)

317430clone = fmap unsafeFreeze . MA.clone . unsafeThaw

318431

319432{-# INLINE pinnedClone #-}

320−pinnedClone :: MonadIO m => Array a -> m (Array a)

433+pinnedClone ::

434+ ( MonadIO m

435+#ifdef DEVBUILD

436+ , Unbox a

437+#endif

438+ )

439+ => Array a -> m (Array a)

321440pinnedClone = fmap unsafeFreeze . MA.pinnedClone . unsafeThaw

322441

323442-------------------------------------------------------------------------------

324443-- Streams of arrays

325444-------------------------------------------------------------------------------

326445

327−{-# INLINE bufferChunks #-}

446+{-# INLINE buildChunks #-}

447+buildChunks :: (MonadIO m, Unbox a) =>

448+ D.Stream m a -> m (K.StreamK m (Array a))

449+buildChunks m = D.foldr K.cons K.nil $ chunksOf defaultChunkSize m

450+

451+{-# DEPRECATED bufferChunks "Please use buildChunks instead." #-}

328452bufferChunks :: (MonadIO m, Unbox a) =>

329453 D.Stream m a -> m (K.StreamK m (Array a))

330−bufferChunks m = D.foldr K.cons K.nil $ chunksOf defaultChunkSize m

454+bufferChunks = buildChunks

331455

332456-- | @chunksOf n stream@ groups the elements in the input stream into arrays of

333457-- @n@ elements each.

@@ -348,27 +472,100 @@ pinnedChunksOf :: forall m a. (MonadIO m, Unbox a)

348472 => Int -> D.Stream m a -> D.Stream m (Array a)

349473pinnedChunksOf n str = D.map unsafeFreeze $ MA.pinnedChunksOf n str

350474

351−-- | Use the "read" unfold instead.

475+-- | Convert a stream of arrays into a stream of their elements.

352476--

353−-- @flattenArrays = unfoldMany read@

477+-- >>> concat = Stream.unfoldMany Array.reader

354478--

355−-- We can try this if there are any fusion issues in the unfold.

356−--

357−{-# INLINE_NORMAL flattenArrays #-}

479+{-# INLINE_NORMAL concat #-}

480+concat :: (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

481+-- XXX this requires MonadIO whereas the unfoldMany version does not

482+concat = MA.concatWith (pure . unsafeInlineIO) . D.map unsafeThaw

483+-- concat = D.unfoldMany reader

484+

485+{-# DEPRECATED flattenArrays "Please use \"unfoldMany reader\" instead." #-}

486+{-# INLINE flattenArrays #-}

358487flattenArrays :: forall m a. (MonadIO m, Unbox a)

359488 => D.Stream m (Array a) -> D.Stream m a

360−flattenArrays = MA.flattenArrays . D.map unsafeThaw

489+flattenArrays = concat

361490

362−-- | Use the "readRev" unfold instead.

363−--

364−-- @flattenArrays = unfoldMany readRev@

491+-- | Convert a stream of arrays into a stream of their elements reversing the

492+-- contents of each array before flattening.

365493--

366−-- We can try this if there are any fusion issues in the unfold.

494+-- >>> concatRev = Stream.unfoldMany Array.readerRev

367495--

368−{-# INLINE_NORMAL flattenArraysRev #-}

496+{-# INLINE_NORMAL concatRev #-}

497+concatRev :: forall m a. (Monad m, Unbox a)

498+ => D.Stream m (Array a) -> D.Stream m a

499+-- XXX this requires MonadIO whereas the unfoldMany version does not

500+concatRev = MA.concatRevWith (pure . unsafeInlineIO) . D.map unsafeThaw

501+-- concatRev = D.unfoldMany readerRev

502+

503+{-# DEPRECATED flattenArraysRev "Please use \"unfoldMany readerRev\" instead." #-}

504+{-# INLINE flattenArraysRev #-}

369505flattenArraysRev :: forall m a. (MonadIO m, Unbox a)

370506 => D.Stream m (Array a) -> D.Stream m a

371−flattenArraysRev = MA.flattenArraysRev . D.map unsafeThaw

507+flattenArraysRev = concatRev

508+

509+-------------------------------------------------------------------------------

510+-- Compact

511+-------------------------------------------------------------------------------

512+

513+-- XXX Note that this thaws immutable arrays for appending, that may be

514+-- problematic if multiple users do the same thing, however, thawed immutable

515+-- arrays would have no capacity to append, therefore, a copy will be forced

516+-- anyway.

517+

518+-- | Fold @fCompactGE n@ coalesces adjacent arrays in the input stream

519+-- until the size becomes greater than or equal to n.

520+--

521+-- Generates unpinned arrays irrespective of the pinning status of input

522+-- arrays.

523+{-# INLINE_NORMAL fCompactGE #-}

524+fCompactGE :: (MonadIO m, Unbox a) => Int -> Fold m (Array a) (Array a)

525+fCompactGE n = fmap unsafeFreeze $ Fold.lmap unsafeThaw $ MA.fCompactGE n

526+

527+-- | PInned version of 'fCompactGE'.

528+{-# INLINE_NORMAL fPinnedCompactGE #-}

529+fPinnedCompactGE :: (MonadIO m, Unbox a) => Int -> Fold m (Array a) (Array a)

530+fPinnedCompactGE n =

531+ fmap unsafeFreeze $ Fold.lmap unsafeThaw $ MA.fPinnedCompactGE n

532+

533+-- | @compactGE n stream@ coalesces adjacent arrays in the @stream@ until

534+-- the size becomes greater than or equal to @n@.

535+--

536+-- >>> compactGE n = Stream.foldMany (Array.fCompactGE n)

537+--

538+-- Generates unpinned arrays irrespective of the pinning status of input

539+-- arrays.

540+{-# INLINE compactGE #-}

541+compactGE ::

542+ (MonadIO m, Unbox a)

543+ => Int -> Stream m (Array a) -> Stream m (Array a)

544+compactGE n stream =

545+ D.map unsafeFreeze $ MA.compactGE n $ D.map unsafeThaw stream

546+

547+-- | Like 'compactGE' but for transforming folds instead of stream.

548+--

549+-- >>> lCompactGE n = Fold.many (Array.fCompactGE n)

550+--

551+-- Generates unpinned arrays irrespective of the pinning status of input

552+-- arrays.

553+{-# INLINE_NORMAL lCompactGE #-}

554+lCompactGE :: (MonadIO m, Unbox a)

555+ => Int -> Fold m (Array a) () -> Fold m (Array a) ()

556+lCompactGE n fld =

557+ Fold.lmap unsafeThaw $ MA.lCompactGE n (Fold.lmap unsafeFreeze fld)

558+

559+-- | Pinned version of 'lCompactGE'.

560+{-# INLINE_NORMAL lPinnedCompactGE #-}

561+lPinnedCompactGE :: (MonadIO m, Unbox a)

562+ => Int -> Fold m (Array a) () -> Fold m (Array a) ()

563+lPinnedCompactGE n fld =

564+ Fold.lmap unsafeThaw $ MA.lPinnedCompactGE n (Fold.lmap unsafeFreeze fld)

565+

566+-------------------------------------------------------------------------------

567+-- Splitting

568+-------------------------------------------------------------------------------

372569

373570-- Drops the separator byte

374571{-# INLINE breakOn #-}

@@ -412,24 +609,68 @@ byteLength = MA.byteLength . unsafeThaw

412609length :: Unbox a => Array a -> Int

413610length arr = MA.length (unsafeThaw arr)

414611

612+{-# INLINE_NORMAL producer #-}

613+producer :: forall m a. (Monad m, Unbox a) => Producer m (Array a) a

614+producer =

615+ Producer.translate unsafeThaw unsafeFreeze

616+ $ MA.producerWith (return . unsafeInlineIO)

617+

618+-- | Unfold an array into a stream.

619+--

620+{-# INLINE_NORMAL reader #-}

621+reader :: forall m a. (Monad m, Unbox a) => Unfold m (Array a) a

622+reader = Producer.simplify producer

623+

624+-- | Unfold an array into a stream, does not check the end of the array, the

625+-- user is responsible for terminating the stream within the array bounds. For

626+-- high performance application where the end condition can be determined by

627+-- a terminating fold.

628+--

629+-- Written in the hope that it may be faster than "read", however, in the case

630+-- for which this was written, "read" proves to be faster even though the core

631+-- generated with unsafeRead looks simpler.

632+--

633+-- /Pre-release/

634+--

635+{-# INLINE_NORMAL readerUnsafe #-}

636+readerUnsafe :: forall m a. (Monad m, Unbox a) => Unfold m (Array a) a

637+readerUnsafe = Unfold step inject

638+ where

639+

640+ inject (Array contents start end) =

641+ return (MA.ArrayUnsafe contents end start)

642+

643+ {-# INLINE_LATE step #-}

644+ step (MA.ArrayUnsafe contents end p) = do

645+ -- unsafeInlineIO allows us to run this in Identity monad for pure

646+ -- toList/foldr case which makes them much faster due to not

647+ -- accumulating the list and fusing better with the pure consumers.

648+ --

649+ -- This should be safe as the array contents are guaranteed to be

650+ -- evaluated/written to before we peek at them.

651+ let !x = unsafeInlineIO $ peekAt p contents

652+ let !p1 = INDEX_NEXT(p,a)

653+ return $ D.Yield x (MA.ArrayUnsafe contents end p1)

654+

415655-- | Unfold an array into a stream in reverse order.

416656--

417657{-# INLINE_NORMAL readerRev #-}

418658readerRev :: forall m a. (Monad m, Unbox a) => Unfold m (Array a) a

419659readerRev = Unfold.lmap unsafeThaw $ MA.readerRevWith (return . unsafeInlineIO)

420660

661+{-# DEPRECATED toStreamD "Please use 'read' instead." #-}

421662{-# INLINE_NORMAL toStreamD #-}

422663toStreamD :: forall m a. (Monad m, Unbox a) => Array a -> D.Stream m a

423−toStreamD arr = MA.toStreamDWith (return . unsafeInlineIO) (unsafeThaw arr)

664+toStreamD = read

424665

425666{-# INLINE toStreamK #-}

426667toStreamK :: forall m a. (Monad m, Unbox a) => Array a -> K.StreamK m a

427668toStreamK arr = MA.toStreamKWith (return . unsafeInlineIO) (unsafeThaw arr)

428669

670+{-# DEPRECATED toStreamDRev "Please use 'readRev' instead." #-}

429671{-# INLINE_NORMAL toStreamDRev #-}

430672toStreamDRev :: forall m a. (Monad m, Unbox a) => Array a -> D.Stream m a

431−toStreamDRev arr =

432− MA.toStreamDRevWith (return . unsafeInlineIO) (unsafeThaw arr)

673+toStreamDRev = readRev

433674

434675{-# INLINE toStreamKRev #-}

435676toStreamKRev :: forall m a. (Monad m, Unbox a) => Array a -> K.StreamK m a

@@ -441,7 +682,7 @@ toStreamKRev arr =

441682-- /Pre-release/

442683{-# INLINE_EARLY read #-}

443684read :: (Monad m, Unbox a) => Array a -> Stream m a

444−read = toStreamD

685+read arr = MA.toStreamWith (return . unsafeInlineIO) (unsafeThaw arr)

445686

446687-- | Same as 'read'

447688--

@@ -458,7 +699,7 @@ toStream = read

458699-- /Pre-release/

459700{-# INLINE_EARLY readRev #-}

460701readRev :: (Monad m, Unbox a) => Array a -> Stream m a

461−readRev = toStreamDRev

702+readRev arr = MA.toStreamRevWith (return . unsafeInlineIO) (unsafeThaw arr)

462703

463704-- | Same as 'readRev'

464705--

@@ -482,6 +723,7 @@ foldr f z arr = runIdentity $ D.foldr f z $ toStreamD arr

482723-- | Create two slices of an array without copying the original array. The

483724-- specified index @i@ is the first index of the second slice.

484725--

726+{-# INLINE splitAt #-}

485727splitAt :: Unbox a => Int -> Array a -> (Array a, Array a)

486728splitAt i arr = (unsafeFreeze a, unsafeFreeze b)

487729 where

@@ -501,8 +743,8 @@ toListFB c n Array{..} = go arrStart

501743 -- accumulating the list and fusing better with the pure consumers.

502744 --

503745 -- This should be safe as the array contents are guaranteed to be

504− -- evaluated/written to before we peekByteIndex at them.

505− let !x = unsafeInlineIO $ peekByteIndex p arrContents

746+ -- evaluated/written to before we peekAt at them.

747+ let !x = unsafeInlineIO $ peekAt p arrContents

506748 in c x (go (INDEX_NEXT(p,a)))

507749

508750-- | Convert an 'Array' into a list.

@@ -515,17 +757,28 @@ toList s = build (\c n -> toListFB c n s)

515757-- Folds

516758-------------------------------------------------------------------------------

517759

518−-- | @writeN n@ folds a maximum of @n@ elements from the input stream to an

760+-- | @createOf n@ folds a maximum of @n@ elements from the input stream to an

519761-- 'Array'.

520762--

521−{-# INLINE_NORMAL writeN #-}

763+{-# INLINE_NORMAL createOf #-}

764+createOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

765+createOf = fmap unsafeFreeze . MA.createOf

766+

767+-- XXX Deprecate in major

768+-- {-# DEPRECATED writeN "Please use createOf instead." #-}

769+{-# INLINE writeN #-}

522770writeN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

523−writeN = fmap unsafeFreeze . MA.writeN

771+writeN = createOf

524772

525−-- | Like 'fromListN' but creates a pinned array.

526−{-# INLINE_NORMAL pinnedWriteN #-}

773+-- | Like 'createOf' but creates a pinned array.

774+{-# INLINE_NORMAL pinnedCreateOf #-}

775+pinnedCreateOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

776+pinnedCreateOf = fmap unsafeFreeze . MA.pinnedCreateOf

777+

778+{-# DEPRECATED pinnedWriteN "Please use pinnedCreateOf instead." #-}

779+{-# INLINE pinnedWriteN #-}

527780pinnedWriteN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (Array a)

528−pinnedWriteN = fmap unsafeFreeze . MA.pinnedWriteN

781+pinnedWriteN = pinnedCreateOf

529782

530783-- | @pinnedWriteNAligned alignment n@ folds a maximum of @n@ elements from the input

531784-- stream to an 'Array' aligned to the given size.

@@ -533,44 +786,74 @@ pinnedWriteN = fmap unsafeFreeze . MA.pinnedWriteN

533786-- /Pre-release/

534787--

535788{-# INLINE_NORMAL pinnedWriteNAligned #-}

789+{-# DEPRECATED pinnedWriteNAligned "To be removed." #-}

536790pinnedWriteNAligned :: forall m a. (MonadIO m, Unbox a)

537791 => Int -> Int -> Fold m a (Array a)

538792pinnedWriteNAligned alignSize = fmap unsafeFreeze . MA.pinnedWriteNAligned alignSize

539793

540−-- | Like 'writeN' but does not check the array bounds when writing. The fold

794+-- | Like 'createOf' but does not check the array bounds when writing. The fold

541795-- driver must not call the step function more than 'n' times otherwise it will

542796-- corrupt the memory and crash. This function exists mainly because any

543797-- conditional in the step function blocks fusion causing 10x performance

544798-- slowdown.

545799--

546−{-# INLINE_NORMAL writeNUnsafe #-}

800+{-# INLINE_NORMAL unsafeCreateOf #-}

801+unsafeCreateOf :: forall m a. (MonadIO m, Unbox a)

802+ => Int -> Fold m a (Array a)

803+unsafeCreateOf n = unsafeFreeze <$> MA.unsafeCreateOf n

804+

805+{-# DEPRECATED writeNUnsafe "Please use unsafeCreateOf instead." #-}

806+{-# INLINE writeNUnsafe #-}

547807writeNUnsafe :: forall m a. (MonadIO m, Unbox a)

548808 => Int -> Fold m a (Array a)

549−writeNUnsafe n = unsafeFreeze <$> MA.writeNUnsafe n

809+writeNUnsafe = unsafeCreateOf

550810

551−{-# INLINE_NORMAL pinnedWriteNUnsafe #-}

811+{-# INLINE_NORMAL unsafePinnedCreateOf #-}

812+unsafePinnedCreateOf :: forall m a. (MonadIO m, Unbox a)

813+ => Int -> Fold m a (Array a)

814+unsafePinnedCreateOf n = unsafeFreeze <$> MA.unsafePinnedCreateOf n

815+

816+{-# DEPRECATED pinnedWriteNUnsafe "Please use unsafePinnedCreateOf instead." #-}

817+{-# INLINE pinnedWriteNUnsafe #-}

552818pinnedWriteNUnsafe :: forall m a. (MonadIO m, Unbox a)

553819 => Int -> Fold m a (Array a)

554−pinnedWriteNUnsafe n = unsafeFreeze <$> MA.pinnedWriteNUnsafe n

820+pinnedWriteNUnsafe = unsafePinnedCreateOf

821+

822+{-# INLINE_NORMAL createWith #-}

823+createWith :: forall m a. (MonadIO m, Unbox a)

824+ => Int -> Fold m a (Array a)

825+-- createWith n = FL.rmapM spliceArrays $ toArraysOf n

826+createWith elemCount = unsafeFreeze <$> MA.createWith elemCount

555827

556−{-# INLINE_NORMAL writeWith #-}

828+{-# DEPRECATED writeWith "Please use createWith instead." #-}

829+{-# INLINE writeWith #-}

557830writeWith :: forall m a. (MonadIO m, Unbox a)

558831 => Int -> Fold m a (Array a)

559−-- writeWith n = FL.rmapM spliceArrays $ toArraysOf n

560−writeWith elemCount = unsafeFreeze <$> MA.writeWith elemCount

832+writeWith = createWith

561833

562834-- | Fold the whole input to a single array.

563835--

564836-- /Caution! Do not use this on infinite streams./

565837--

838+{-# INLINE create #-}

839+create :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

840+create = fmap unsafeFreeze MA.create

841+

842+-- XXX Deprecate in major

843+-- {-# DEPRECATED write "Please use create instead." #-}

566844{-# INLINE write #-}

567845write :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

568−write = fmap unsafeFreeze MA.write

846+write = create

569847

570−-- | Like 'write' but creates a pinned array.

848+-- | Like 'create' but creates a pinned array.

849+{-# INLINE pinnedCreate #-}

850+pinnedCreate :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

851+pinnedCreate = fmap unsafeFreeze MA.pinnedCreate

852+

853+{-# DEPRECATED pinnedWrite "Please use pinnedCreate instead." #-}

571854{-# INLINE pinnedWrite #-}

572855pinnedWrite :: forall m a. (MonadIO m, Unbox a) => Fold m a (Array a)

573−pinnedWrite = fmap unsafeFreeze MA.pinnedWrite

856+pinnedWrite = pinnedCreate

574857

575858-- | Fold "step" has a dependency on "initial", and each step is dependent on

576859-- the previous invocation of step due to state passing, finally extract

@@ -602,6 +885,11 @@ unsafeMakePure (Fold step initial extract final) =

602885 (\s -> return $! unsafeInlineIO $ extract s)

603886 (\s -> return $! unsafeInlineIO $ final s)

604887

888+{-# INLINE fromPureStreamN #-}

889+fromPureStreamN :: Unbox a => Int -> Stream Identity a -> Array a

890+fromPureStreamN n x =

891+ unsafePerformIO $ fmap unsafeFreeze (MA.fromPureStreamN n x)

892+

605893-- | Convert a pure stream in Identity monad to an immutable array.

606894--

607895-- Same as the following but with better performance:

@@ -613,6 +901,19 @@ fromPureStream x = unsafePerformIO $ fmap (unsafeFreeze) (MA.fromPureStream x)

613901-- fromPureStream = runIdentity . D.fold (unsafeMakePure write)

614902-- fromPureStream = fromList . runIdentity . D.toList

615903

904+-- XXX This should be monadic.

905+

906+-- | Copy an immutable 'Ptr Word8' sequence into an array.

907+--

908+-- /Unsafe:/ The caller is responsible for safe addressing.

909+--

910+-- Note that this should be evaluated strictly to ensure that we do not hold

911+-- the reference to the pointer in a lazy thunk.

912+fromPtrN :: Int -> Ptr Word8 -> Array Word8

913+fromPtrN n addr = unsafePerformIO $ fmap unsafeFreeze (MA.fromPtrN n addr)

914+

915+-- XXX This should be monadic.

916+

616917-- | Copy a null terminated immutable 'Addr#' Word8 sequence into an array.

617918--

618919-- /Unsafe:/ The caller is responsible for safe addressing.

@@ -623,8 +924,46 @@ fromPureStream x = unsafePerformIO $ fmap (unsafeFreeze) (MA.fromPureStream x)

623924-- >>> Array.toList $ Array.fromByteStr# "\1\2\3\0"#

624925-- [1,2,3]

625926--

927+-- Note that this should be evaluated strictly to ensure that we do not hold

928+-- the reference to the pointer in a lazy thunk.

626929fromByteStr# :: Addr# -> Array Word8

627−fromByteStr# addr = fromPureStream (D.fromByteStr# addr)

930+fromByteStr# addr = unsafePerformIO $ fmap unsafeFreeze (MA.fromByteStr# addr)

931+

932+-- XXX This should be monadic.

933+

934+-- | Note that this should be evaluated strictly to ensure that we do not hold

935+-- the reference to the pointer in a lazy thunk.

936+fromByteStr :: Ptr Word8 -> Array Word8

937+fromByteStr (Ptr addr#) = fromByteStr# addr#

938+

939+-- XXX implement fromChunks/fromChunkList instead?

940+

941+-- | Convert an array stream to an array. Note that this requires peak memory

942+-- that is double the size of the array stream.

943+--

944+{-# INLINE fromChunksK #-}

945+fromChunksK :: (MonadIO m, Unbox a) => StreamK m (Array a) -> m (Array a)

946+fromChunksK stream =

947+ -- We buffer the entire stream and then allocate the target array of the

948+ -- same size, thus requiring double the memory.

949+ fmap unsafeFreeze $ MA.fromChunksK $ fmap unsafeThaw stream

950+

951+{-# DEPRECATED fromArrayStreamK "Please use fromChunksK instead." #-}

952+fromArrayStreamK :: (Unbox a, MonadIO m) => StreamK m (Array a) -> m (Array a)

953+fromArrayStreamK = fromChunksK

954+

955+-- | Given a stream of arrays, splice them all together to generate a single

956+-- array. The stream must be /finite/.

957+--

958+{-# INLINE fromChunks #-}

959+fromChunks :: (MonadIO m, Unbox a) => Stream m (Array a) -> m (Array a)

960+fromChunks s =

961+ -- XXX Check which implementation is better

962+ -- This may also require double the memory as we double the space every

963+ -- time, when copying the last array we may have reallocated almost double

964+ -- the space required before we right size it.

965+ fmap unsafeFreeze $ MA.fromChunksRealloced (fmap unsafeThaw s)

966+ -- fromChunkStreamK $ D.toStreamK s

628967

629968-------------------------------------------------------------------------------

630969-- Instances

@@ -656,12 +995,66 @@ instance Unbox a => IsList (Array a) where

656995 {-# INLINE toList #-}

657996 toList = toList

658997

659−-- XXX we are assuming that Unboxed equality means element equality. This may

660−-- or may not be correct? arrcmp is 40% faster compared to stream equality.

661−instance (Unbox a, Eq a) => Eq (Array a) where

998+-- | Byte compare two arrays. Compare the length of the arrays. If the length

999+-- is equal, compare the lexicographical ordering of two underlying byte arrays

1000+-- otherwise return the result of length comparison.

1001+--

1002+-- /Unsafe/: Note that the 'Unbox' instance of sum types with constructors of

1003+-- different sizes may leave some memory uninitialized which can make byte

1004+-- comparison unreliable.

1005+--

1006+-- /Pre-release/

1007+{-# INLINE byteCmp #-}

1008+byteCmp :: Array a -> Array a -> Ordering

1009+byteCmp arr1 arr2 =

1010+ -- unsafePerformIO?

1011+ unsafeInlineIO $! unsafeThaw arr1 `MA.byteCmp` unsafeThaw arr2

1012+

1013+-- | Byte equality of two arrays.

1014+--

1015+-- >>> byteEq arr1 arr2 = (==) EQ $ Array.byteCmp arr1 arr2

1016+--

1017+-- /Unsafe/: See 'byteCmp'.

1018+{-# INLINE byteEq #-}

1019+byteEq :: Array a -> Array a -> Bool

1020+byteEq arr1 arr2 = (==) EQ $ byteCmp arr1 arr2

1021+

1022+#define MK_EQ_INSTANCE(typ) \

1023+instance {-# OVERLAPPING #-} Eq (Array typ) where { \

1024+; {-# INLINE (==) #-} \

1025+; (==) = byteEq \

1026+}

1027+

1028+MK_EQ_INSTANCE(Char)

1029+MK_EQ_INSTANCE(Word8)

1030+MK_EQ_INSTANCE(Word16)

1031+MK_EQ_INSTANCE(Word32)

1032+

1033+-- XXX The Word64 default instance should be as fast because we are comparing

1034+-- 64-bit at a time.

1035+MK_EQ_INSTANCE(Word64)

1036+MK_EQ_INSTANCE(Int)

1037+MK_EQ_INSTANCE(Int8)

1038+MK_EQ_INSTANCE(Int16)

1039+MK_EQ_INSTANCE(Int32)

1040+

1041+-- XXX The Int64 default instance should be as fast.

1042+MK_EQ_INSTANCE(Int64)

1043+

1044+-- | If the type allows a byte-by-byte comparison this instance can be

1045+-- overlapped by a more specific instance that uses 'byteCmp'. Byte comparison

1046+-- can be significantly faster.

1047+--

1048+instance {-# OVERLAPPABLE #-} (Unbox a, Eq a) => Eq (Array a) where

6621049 {-# INLINE (==) #-}

6631050 arr1 == arr2 =

664− (==) EQ $ unsafeInlineIO $! unsafeThaw arr1 `MA.cmp` unsafeThaw arr2

1051+ -- Does unboxed byte equality mean element equality?

1052+ -- XXX This is incorrect for sum types, as we may have some

1053+ -- uninitialized memory in that case. If we always initialize the

1054+ -- unused memory to zero we can use this.

1055+ -- Byte comparison is 40% faster compared to stream equality.

1056+ -- (==) EQ $ unsafeInlineIO $! unsafeThaw arr1 `MA.cmp` unsafeThaw arr2

1057+ (toStreamD arr1 :: Stream Identity a) == toStreamD arr2

6651058

6661059instance (Unbox a, Ord a) => Ord (Array a) where

6671060 {-# INLINE compare #-}

@@ -703,7 +1096,7 @@ _toStreamD_ size Array{..} = D.Stream step arrStart

7031096 {-# INLINE_LATE step #-}

7041097 step _ p | p == arrEnd = return D.Stop

7051098 step _ p = liftIO $ do

706− x <- peekByteIndex p arrContents

1099+ x <- peekAt p arrContents

7071100 return $ D.Yield x (p + size)

7081101

7091102{-

@@ -726,15 +1119,29 @@ instance Foldable Array where

7261119-- Semigroup and Monoid

7271120-------------------------------------------------------------------------------

7281121

1122+-- XXX Deprecate and remove the Semigroup and Monoid instances because of

1123+-- potential misuse chances.

1124+

1125+-- | This should not be used for combining many or N arrays as it would copy

1126+-- the two arrays everytime to a new array. For coalescing multiple arrays use

1127+-- 'fromChunksK' instead.

7291128instance Unbox a => Semigroup (Array a) where

7301129 arr1 <> arr2 = unsafePerformIO $ splice arr1 arr2

7311130

1131+empty ::

1132+#ifdef DEVBUILD

1133+ Unbox a =>

1134+#endif

1135+ Array a

1136+empty = Array Unboxed.empty 0 0

1137+

1138+{-# DEPRECATED nil "Please use empty instead." #-}

7321139nil ::

7331140#ifdef DEVBUILD

7341141 Unbox a =>

7351142#endif

7361143 Array a

737−nil = Array Unboxed.nil 0 0

1144+nil = empty

7381145

7391146instance Unbox a => Monoid (Array a) where

7401147 mempty = nil

core/src/Streamly/Internal/Serialize/FromBytes.hs → core/src/Streamly/Internal/Data/Binary/Parser.hsrenamed+8−2

@@ -1,13 +1,19 @@

11-- |

2−-- Module : Streamly.Internal.Serialize.FromBytes

2+-- Module : Streamly.Internal.Data.Binary.Parser

44-- License : BSD-3-Clause

55-- Maintainer : streamly@composewell.com

66-- Portability : GHC

77--

88-- Decode Haskell data types from byte streams.

9+--

10+-- It would be inefficient to use this to compose parsers for general algebraic

11+-- data types. For general deserialization of ADTs please use the Serialize

12+-- type class instances. The fastest way to deserialize byte streams

13+-- representing Haskell data types is to write them to arrays and deserialize

14+-- the array using the Serialize type class.

915

10−module Streamly.Internal.Serialize.FromBytes

16+module Streamly.Internal.Data.Binary.Parser

1117 (

1218 -- * Type class

1319 FromBytes (..)

core/src/Streamly/Internal/Serialize/ToBytes.hs → core/src/Streamly/Internal/Data/Binary/Stream.hsrenamed+2−2

@@ -1,5 +1,5 @@

11-- |

2−-- Module : Streamly.Internal.Serialize.ToBytes

2+-- Module : Streamly.Internal.Data.Binary.Stream

44-- License : BSD-3-Clause

55-- Maintainer : streamly@composewell.com

@@ -17,7 +17,7 @@

1717

1818-- XXX remove unit, bool, ordering, and the type class as well

1919

20−module Streamly.Internal.Serialize.ToBytes

20+module Streamly.Internal.Data.Binary.Stream

2121 (

2222 -- * Type class

2323 ToBytes (..)

core/src/Streamly/Internal/Data/Fold/Chunked.hs+12−13

@@ -6,7 +6,7 @@

66-- Stability : experimental

77-- Portability : GHC

88--

9−-- Use "Streamly.Data.Parser.Chunked" instead.

9+-- Use "Streamly.Data.Parser" instead.

1010--

1111-- Fold a stream of foreign arrays. @Fold m a b@ in this module works

1212-- on a stream of "Array a" and produces an output of type @b@.

@@ -19,18 +19,19 @@

1919-- folds in Data.Fold to correctly work on an array stream as if it is an

2020-- element stream. For example:

2121--

22−-- >>> import qualified Streamly.Data.Fold as Fold

23−-- >>> import qualified Streamly.Internal.Data.Stream.Chunked as ArrayStream

24−-- >>> import qualified Streamly.Internal.Data.Fold.Chunked as ChunkFold

25−-- >>> import qualified Streamly.Data.Stream as Stream

26−-- >>> import qualified Streamly.Data.StreamK as StreamK

22+-- >> import qualified Streamly.Data.Fold as Fold

23+-- >> import qualified Streamly.Internal.Data.Array.Stream as ArrayStream

24+-- >> import qualified Streamly.Internal.Data.Fold.Chunked as ChunkFold

25+-- >> import qualified Streamly.Data.Stream as Stream

26+-- >> import qualified Streamly.Data.StreamK as StreamK

2727--

28−-- >>> f = ChunkFold.fromFold (Fold.take 7 Fold.toList)

29−-- >>> s = Stream.chunksOf 5 $ Stream.fromList "hello world"

30−-- >>> ArrayStream.runArrayFold f (StreamK.fromStream s)

28+-- >> f = ChunkFold.fromFold (Fold.take 7 Fold.toList)

29+-- >> s = Stream.chunksOf 5 $ Stream.fromList "hello world"

30+-- >> ArrayStream.runArrayFold f (StreamK.fromStream s)

3131-- Right "hello w"

3232--

3333module Streamly.Internal.Data.Fold.Chunked

34+ {-# DEPRECATED "Please use Streamly.Data.Parser instead." #-}

3435 (

3536 ChunkFold (..)

3637

@@ -67,7 +68,6 @@ import Data.Bifunctor (first)

6768import Data.Proxy (Proxy(..))

6869import Streamly.Internal.Data.Unbox (Unbox(..))

6970import GHC.Types (SPEC(..))

70−import Streamly.Internal.Data.MutArray.Type (touch)

7171import Streamly.Internal.Data.Array (Array(..))

7272import Streamly.Internal.Data.Parser (Initial(..), Step(..))

7373import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))

@@ -124,7 +124,7 @@ fromFold (Fold.Fold fstep finitial _ ffinal) =

124124 assert (cur == end) (return ())

125125 return $ Partial 0 fs

126126 goArray !_ !cur !fs = do

127− x <- liftIO $ peekByteIndex cur contents

127+ x <- liftIO $ peekAt cur contents

128128 res <- fstep fs x

129129 let elemSize = SIZE_OF(a)

130130 next = INDEX_NEXT(cur,a)

@@ -163,8 +163,7 @@ fromParserD (ParserD.Parser step1 initial1 extract1) =

163163 else return $ st (arrRem + n) fs1

164164

165165 goArray !_ !cur !fs = do

166− x <- liftIO $ peekByteIndex cur contents

167− liftIO $ touch contents

166+ x <- liftIO $ peekAt cur contents

168167 res <- step1 fs x

169168 let elemSize = SIZE_OF(a)

170169 next = INDEX_NEXT(cur,a)

core/src/Streamly/Internal/Data/Fold/Combinators.hs−92

@@ -30,8 +30,6 @@ module Streamly.Internal.Data.Fold.Combinators

3030 -- *** Reducers

3131 , drainMapM

3232 , the

33− , length

34− , lengthGeneric

3533 , mean

3634 , rollingHash

3735 , defaultSalt

@@ -166,8 +164,6 @@ module Streamly.Internal.Data.Fold.Combinators

166164 -- ** Trimming

167165

168166 -- By elements

169− , takeEndBy

170− , takeEndBy_

171167 , takeEndBySeq

172168 , takeEndBySeq_

173169 {-

@@ -709,30 +705,6 @@ the = foldt' step initial id

709705-- To Summary

710706------------------------------------------------------------------------------

711707

712−-- | Like 'length', except with a more general 'Num' return value

713−--

714−-- Definition:

715−--

716−-- >>> lengthGeneric = fmap getSum $ Fold.foldMap (Sum . const 1)

717−-- >>> lengthGeneric = Fold.foldl' (\n _ -> n + 1) 0

718−--

719−-- /Pre-release/

720−{-# INLINE lengthGeneric #-}

721−lengthGeneric :: (Monad m, Num b) => Fold m a b

722−lengthGeneric = foldl' (\n _ -> n + 1) 0

723−

724−-- | Determine the length of the input stream.

725−--

726−-- Definition:

727−--

728−-- >>> length = Fold.lengthGeneric

729−-- >>> length = fmap getSum $ Fold.foldMap (Sum . const 1)

730−--

731−{-# INLINE length #-}

732−length :: Monad m => Fold m a Int

733−length = lengthGeneric

734−

735−

736708-- | Determine the sum of all elements of a stream of numbers. Returns additive

737709-- identity (@0@) when the stream is empty. Note that this is not numerically

738710-- stable for floating point numbers.

@@ -1510,70 +1482,6 @@ droppingWhileM p = Fold step initial extract extract

15101482droppingWhile :: Monad m => (a -> Bool) -> Fold m a (Maybe a)

15111483droppingWhile p = droppingWhileM (return . p)

15121484

1513−-- Note: Keep this consistent with S.splitOn. In fact we should eliminate

1514−-- S.splitOn in favor of the fold.

1515−--

1516−-- XXX Use Fold.many instead once it is fixed.

1517−-- > Stream.splitOnSuffix p f = Stream.foldMany (Fold.takeEndBy_ p f)

1518−

1519−-- | Like 'takeEndBy' but drops the element on which the predicate succeeds.

1520−--

1521−-- Example:

1522−--

1523−-- >>> input = Stream.fromList "hello\nthere\n"

1524−-- >>> line = Fold.takeEndBy_ (== '\n') Fold.toList

1525−-- >>> Stream.fold line input

1526−-- "hello"

1527−--

1528−-- >>> Stream.fold Fold.toList $ Stream.foldMany line input

1529−-- ["hello","there"]

1530−--

1531−{-# INLINE takeEndBy_ #-}

1532−takeEndBy_ :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b

1533−-- takeEndBy_ predicate = scanMaybe (takingEndBy_ predicate)

1534−takeEndBy_ predicate (Fold fstep finitial fextract ffinal) =

1535− Fold step finitial fextract ffinal

1536−

1537− where

1538−

1539− step s a =

1540− if not (predicate a)

1541− then fstep s a

1542− else Done <$> ffinal s

1543−

1544−-- Note:

1545−-- > Stream.splitWithSuffix p f = Stream.foldMany (Fold.takeEndBy p f)

1546−

1547−-- | Take the input, stop when the predicate succeeds taking the succeeding

1548−-- element as well.

1549−--

1550−-- Example:

1551−--

1552−-- >>> input = Stream.fromList "hello\nthere\n"

1553−-- >>> line = Fold.takeEndBy (== '\n') Fold.toList

1554−-- >>> Stream.fold line input

1555−-- "hello\n"

1556−--

1557−-- >>> Stream.fold Fold.toList $ Stream.foldMany line input

1558−-- ["hello\n","there\n"]

1559−--

1560−{-# INLINE takeEndBy #-}

1561−takeEndBy :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b

1562−-- takeEndBy predicate = scanMaybe (takingEndBy predicate)

1563−takeEndBy predicate (Fold fstep finitial fextract ffinal) =

1564− Fold step finitial fextract ffinal

1565−

1566− where

1567−

1568− step s a = do

1569− res <- fstep s a

1570− if not (predicate a)

1571− then return res

1572− else do

1573− case res of

1574− Partial s1 -> Done <$> ffinal s1

1575− Done b -> return $ Done b

1576−

15771485------------------------------------------------------------------------------

15781486-- Binary splitting on a separator

15791487------------------------------------------------------------------------------

core/src/Streamly/Internal/Data/Fold/Type.hs+95−2

@@ -367,6 +367,8 @@ module Streamly.Internal.Data.Fold.Type

367367 , toList

368368 , toStreamK

369369 , toStreamKRev

370+ , lengthGeneric

371+ , length

370372

371373 -- * Combinators

372374

@@ -391,6 +393,8 @@ module Streamly.Internal.Data.Fold.Type

391393 -- ** Trimming

392394 , take

393395 , taking

396+ , takeEndBy_

397+ , takeEndBy

394398 , dropping

395399

396400 -- ** Sequential application

@@ -767,6 +771,29 @@ toStreamKRev = foldl' (flip K.cons) K.nil

767771toStreamK :: Monad m => Fold m a (K.StreamK n a)

768772toStreamK = foldr K.cons K.nil

769773

774+-- | Like 'length', except with a more general 'Num' return value

775+--

776+-- Definition:

777+--

778+-- >>> lengthGeneric = fmap getSum $ Fold.foldMap (Sum . const 1)

779+-- >>> lengthGeneric = Fold.foldl' (\n _ -> n + 1) 0

780+--

781+-- /Pre-release/

782+{-# INLINE lengthGeneric #-}

783+lengthGeneric :: (Monad m, Num b) => Fold m a b

784+lengthGeneric = foldl' (\n _ -> n + 1) 0

785+

786+-- | Determine the length of the input stream.

787+--

788+-- Definition:

789+--

790+-- >>> length = Fold.lengthGeneric

791+-- >>> length = fmap getSum $ Fold.foldMap (Sum . const 1)

792+--

793+{-# INLINE length #-}

794+length :: Monad m => Fold m a Int

795+length = lengthGeneric

796+

770797------------------------------------------------------------------------------

771798-- Instances

772799------------------------------------------------------------------------------

@@ -844,8 +871,10 @@ data SeqFoldState sl f sr = SeqFoldL !sl | SeqFoldR !f !sr

844871-- complexity, because each composition adds a new branch that each subsequent

845872-- fold's input element has to traverse, therefore, it cannot scale to a large

846873-- number of compositions. After around 100 compositions the performance starts

847−-- dipping rapidly compared to a CPS style implementation. When you need

848−-- scaling use parser monad instead.

874+-- dipping rapidly compared to a CPS style implementation.

875+--

876+-- For larger number of compositions you can convert the fold to a parser and

877+-- use ParserK.

849878--

850879-- /Time: O(n^2) where n is the number of compositions./

851880--

@@ -1515,6 +1544,70 @@ take n (Fold fstep finitial fextract ffinal) = Fold step initial extract final

15151544

15161545 final (Tuple'Fused _ r) = ffinal r

15171546

1547+-- Note: Keep this consistent with S.splitOn. In fact we should eliminate

1548+-- S.splitOn in favor of the fold.

1549+--

1550+-- XXX Use Fold.many instead once it is fixed.

1551+-- > Stream.splitOnSuffix p f = Stream.foldMany (Fold.takeEndBy_ p f)

1552+

1553+-- | Like 'takeEndBy' but drops the element on which the predicate succeeds.

1554+--

1555+-- Example:

1556+--

1557+-- >>> input = Stream.fromList "hello\nthere\n"

1558+-- >>> line = Fold.takeEndBy_ (== '\n') Fold.toList

1559+-- >>> Stream.fold line input

1560+-- "hello"

1561+--

1562+-- >>> Stream.fold Fold.toList $ Stream.foldMany line input

1563+-- ["hello","there"]

1564+--

1565+{-# INLINE takeEndBy_ #-}

1566+takeEndBy_ :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b

1567+-- takeEndBy_ predicate = scanMaybe (takingEndBy_ predicate)

1568+takeEndBy_ predicate (Fold fstep finitial fextract ffinal) =

1569+ Fold step finitial fextract ffinal

1570+

1571+ where

1572+

1573+ step s a =

1574+ if not (predicate a)

1575+ then fstep s a

1576+ else Done <$> ffinal s

1577+

1578+-- Note:

1579+-- > Stream.splitWithSuffix p f = Stream.foldMany (Fold.takeEndBy p f)

1580+

1581+-- | Take the input, stop when the predicate succeeds taking the succeeding

1582+-- element as well.

1583+--

1584+-- Example:

1585+--

1586+-- >>> input = Stream.fromList "hello\nthere\n"

1587+-- >>> line = Fold.takeEndBy (== '\n') Fold.toList

1588+-- >>> Stream.fold line input

1589+-- "hello\n"

1590+--

1591+-- >>> Stream.fold Fold.toList $ Stream.foldMany line input

1592+-- ["hello\n","there\n"]

1593+--

1594+{-# INLINE takeEndBy #-}

1595+takeEndBy :: Monad m => (a -> Bool) -> Fold m a b -> Fold m a b

1596+-- takeEndBy predicate = scanMaybe (takingEndBy predicate)

1597+takeEndBy predicate (Fold fstep finitial fextract ffinal) =

1598+ Fold step finitial fextract ffinal

1599+

1600+ where

1601+

1602+ step s a = do

1603+ res <- fstep s a

1604+ if not (predicate a)

1605+ then return res

1606+ else do

1607+ case res of

1608+ Partial s1 -> Done <$> ffinal s1

1609+ Done b -> return $ Done b

1610+

15181611------------------------------------------------------------------------------

15191612-- Nesting

15201613------------------------------------------------------------------------------

core/src/Streamly/Internal/Data/Fold/Window.hs+7−1

@@ -65,6 +65,8 @@ import Streamly.Internal.Data.Fold.Type (Fold(..), Step(..))

6565import Streamly.Internal.Data.Tuple.Strict

6666 (Tuple'(..), Tuple3Fused' (Tuple3Fused'))

6767

68+import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)

69+

6870import qualified Streamly.Internal.Data.Fold.Type as Fold

6971import qualified Streamly.Internal.Data.Ring as Ring

7072

@@ -300,7 +302,11 @@ windowRange n = Fold step initial extract extract

300302 if i == 0

301303 then return Nothing

302304 else do

303− x <- liftIO $ peek rh

305+ -- Here we use "ringStart" over "ringHead" as "ringHead" will be

306+ -- uninitialized if the ring is not full.

307+ -- Using "unsafeForeignPtrToPtr" here is safe as we touch the ring

308+ -- again in "foldFunc".

309+ x <- liftIO $ peek (unsafeForeignPtrToPtr (Ring.ringStart rb))

304310 let accum (mn, mx) a = return (min mn a, max mx a)

305311 fmap Just $ foldFunc i rh accum (x, x) rb

306312

core/src/Streamly/Internal/Data/IORef/Unboxed.hs+22−19

@@ -23,34 +23,31 @@ module Streamly.Internal.Data.IORef.Unboxed

2323 (

2424 IORef

2525

26− -- * Construction

26+ -- Construction

2727 , newIORef

2828

29− -- * Write

29+ -- Write

3030 , writeIORef

3131 , modifyIORef'

3232

33− -- * Read

33+ -- Read

3434 , readIORef

35− , toStreamD

35+ , pollIntIORef

3636 )

3737where

3838

3939#include "inline.hs"

4040

41−import Data.Proxy (Proxy(..))

4241import Control.Monad.IO.Class (MonadIO(..))

43−import Streamly.Internal.Data.Unbox

44− ( MutableByteArray(..)

45− , Unbox(..)

46− , newBytes

47− , sizeOf

48− )

42+import Data.Proxy (Proxy(..))

43+import Streamly.Internal.Data.MutByteArray.Type (MutByteArray)

44+import Streamly.Internal.Data.Unbox (Unbox(..), sizeOf)

4945

46+import qualified Streamly.Internal.Data.MutByteArray.Type as MBA

5047import qualified Streamly.Internal.Data.Stream.Type as D

5148

5249-- | An 'IORef' holds a single 'Unbox'-able value.

53−newtype IORef a = IORef MutableByteArray

50+newtype IORef a = IORef MutByteArray

5451

5552-- | Create a new 'IORef'.

5653--

@@ -58,8 +55,8 @@ newtype IORef a = IORef MutableByteArray

5855{-# INLINE newIORef #-}

5956newIORef :: forall a. Unbox a => a -> IO (IORef a)

6057newIORef x = do

61− var <- newBytes (sizeOf (Proxy :: Proxy a))

62− pokeByteIndex 0 var x

58+ var <- MBA.new (sizeOf (Proxy :: Proxy a))

59+ pokeAt 0 var x

6360 return $ IORef var

6461

6562-- | Write a value to an 'IORef'.

@@ -67,14 +64,14 @@ newIORef x = do

6764-- /Pre-release/

6865{-# INLINE writeIORef #-}

6966writeIORef :: Unbox a => IORef a -> a -> IO ()

70−writeIORef (IORef var) = pokeByteIndex 0 var

67+writeIORef (IORef var) = pokeAt 0 var

7168

7269-- | Read a value from an 'IORef'.

7370--

7471-- /Pre-release/

7572{-# INLINE readIORef #-}

7673readIORef :: Unbox a => IORef a -> IO a

77−readIORef (IORef var) = peekByteIndex 0 var

74+readIORef (IORef var) = peekAt 0 var

7875

7976-- | Modify the value of an 'IORef' using a function with strict application.

8077--

@@ -87,10 +84,16 @@ modifyIORef' var g = do

8784

8885-- | Generate a stream by continuously reading the IORef.

8986--

87+-- This operation reads the IORef without any synchronization. It can be

88+-- assumed to be atomic because the IORef (MutableByteArray) is always aligned

89+-- to Int boundaries, we are assuming that compiler uses single instructions to

90+-- access the memory. It may read stale values though until caches are

91+-- synchronised in a multiprocessor architecture.

92+--

9093-- /Pre-release/

91−{-# INLINE_NORMAL toStreamD #-}

92−toStreamD :: (MonadIO m, Unbox a) => IORef a -> D.Stream m a

93−toStreamD var = D.Stream step ()

94+{-# INLINE_NORMAL pollIntIORef #-}

95+pollIntIORef :: (MonadIO m, Unbox a) => IORef a -> D.Stream m a

96+pollIntIORef var = D.Stream step ()

9497

9598 where

9699

core/src/Streamly/Internal/Data/IsMap.hs+4

@@ -1,3 +1,7 @@

1+{-# LANGUAGE TypeFamilies #-}

2+-- Must come after TypeFamilies, otherwise it is re-enabled.

3+-- MonoLocalBinds enabled by TypeFamilies causes perf regressions in general.

4+{-# LANGUAGE NoMonoLocalBinds #-}

15-- |

26-- Module : Streamly.Internal.Data.IsMap

core/src/Streamly/Internal/Data/MutArray.hs+154−33

@@ -8,79 +8,200 @@

88--

99module Streamly.Internal.Data.MutArray

1010 (

11+ -- * MutArray.Type module

1112 module Streamly.Internal.Data.MutArray.Type

12− , splitOn

13+ -- * MutArray module

14+ , sliceIndexerFromLen

15+ , slicerFromLen

16+ , compactLE

17+ , pinnedCompactLE

18+ , compactOnByte

19+ , compactOnByteSuffix

20+ -- * Unboxed IORef

21+ , module Streamly.Internal.Data.IORef.Unboxed

22+

23+ -- * Deprecated

1324 , genSlicesFromLen

1425 , getSlicesFromLen

15− , fromStream

1626 )

1727where

1828

1929#include "inline.hs"

2030

2131import Control.Monad.IO.Class (MonadIO(..))

32+import Data.Word (Word8)

33+import Streamly.Internal.Data.MutByteArray.Type (PinnedState(..))

34+import Streamly.Internal.Data.Stream.Type (Stream)

2235import Streamly.Internal.Data.Unbox (Unbox)

23−import Streamly.Internal.Data.Stream (Stream)

2436import Streamly.Internal.Data.Unfold.Type (Unfold(..))

2537

26−import qualified Streamly.Internal.Data.Stream as D

38+import qualified Streamly.Internal.Data.Stream.Type as Stream

39+import qualified Streamly.Internal.Data.Stream.Nesting as Stream

40+-- import qualified Streamly.Internal.Data.Stream.Transform as Stream

2741import qualified Streamly.Internal.Data.Unfold as Unfold

2842

2943import Prelude hiding (foldr, length, read, splitAt)

3044import Streamly.Internal.Data.MutArray.Type

31−

32−-- | Split the array into a stream of slices using a predicate. The element

33−-- matching the predicate is dropped.

34−--

35−-- /Pre-release/

36−{-# INLINE splitOn #-}

37−splitOn :: (MonadIO m, Unbox a) =>

38− (a -> Bool) -> MutArray a -> Stream m (MutArray a)

39−splitOn predicate arr =

40− fmap (\(i, len) -> getSliceUnsafe i len arr)

41− $ D.sliceOnSuffix predicate (read arr)

45+import Streamly.Internal.Data.IORef.Unboxed

4246

4347-- | Generate a stream of array slice descriptors ((index, len)) of specified

4448-- length from an array, starting from the supplied array index. The last slice

4549-- may be shorter than the requested length depending on the array length.

4650--

4751-- /Pre-release/

48−{-# INLINE genSlicesFromLen #-}

49−genSlicesFromLen :: forall m a. (Monad m, Unbox a)

52+{-# INLINE sliceIndexerFromLen #-}

53+sliceIndexerFromLen :: forall m a. (Monad m, Unbox a)

5054 => Int -- ^ from index

5155 -> Int -- ^ length of the slice

5256 -> Unfold m (MutArray a) (Int, Int)

53−genSlicesFromLen from len =

57+sliceIndexerFromLen from len =

5458 let fromThenTo n = (from, from + len, n - 1)

5559 mkSlice n i = return (i, min len (n - i))

5660 in Unfold.lmap length

5761 $ Unfold.mapM2 mkSlice

5862 $ Unfold.lmap fromThenTo Unfold.enumerateFromThenTo

5963

64+{-# DEPRECATED genSlicesFromLen "Please use sliceIndexerFromLen instead." #-}

65+genSlicesFromLen :: forall m a. (Monad m, Unbox a)

66+ => Int -- ^ from index

67+ -> Int -- ^ length of the slice

68+ -> Unfold m (MutArray a) (Int, Int)

69+genSlicesFromLen = sliceIndexerFromLen

70+

6071-- | Generate a stream of slices of specified length from an array, starting

6172-- from the supplied array index. The last slice may be shorter than the

6273-- requested length depending on the array length.

6374--

6475-- /Pre-release/

65−{-# INLINE getSlicesFromLen #-}

66−getSlicesFromLen :: forall m a. (Monad m, Unbox a)

76+{-# INLINE slicerFromLen #-}

77+slicerFromLen :: forall m a. (Monad m, Unbox a)

6778 => Int -- ^ from index

6879 -> Int -- ^ length of the slice

6980 -> Unfold m (MutArray a) (MutArray a)

70−getSlicesFromLen from len =

81+slicerFromLen from len =

7182 let mkSlice arr (i, n) = return $ getSliceUnsafe i n arr

72− in Unfold.mapM2 mkSlice (genSlicesFromLen from len)

83+ in Unfold.mapM2 mkSlice (sliceIndexerFromLen from len)

84+

85+{-# DEPRECATED getSlicesFromLen "Please use slicerFromLen instead." #-}

86+getSlicesFromLen :: forall m a. (Monad m, Unbox a)

87+ => Int -- ^ from index

88+ -> Int -- ^ length of the slice

89+ -> Unfold m (MutArray a) (MutArray a)

90+getSlicesFromLen = slicerFromLen

7391

74−-- | Create an 'Array' from a stream. This is useful when we want to create a

75−-- single array from a stream of unknown size. 'writeN' is at least twice

76−-- as efficient when the size is already known.

92+-------------------------------------------------------------------------------

93+-- Compacting Streams of Arrays

94+-------------------------------------------------------------------------------

95+

96+-- | @compactLE maxElems@ coalesces adjacent arrays in the input stream

97+-- only if the combined size would be less than or equal to @maxElems@

98+-- elements. Note that it won't split an array if the original array is already

99+-- larger than maxElems.

77100--

78−-- Note that if the input stream is too large memory allocation for the array

79−-- may fail. When the stream size is not known, `chunksOf` followed by

80−-- processing of indvidual arrays in the resulting stream should be preferred.

101+-- @maxElems@ must be greater than 0.

81102--

82−-- /Pre-release/

83−{-# INLINE fromStream #-}

84−fromStream :: (MonadIO m, Unbox a) => Stream m a -> m (MutArray a)

85−fromStream = fromStreamD

86−-- fromStream (Stream m) = P.fold write m

103+-- Generates unpinned arrays irrespective of the pinning status of input

104+-- arrays.

105+{-# INLINE compactLE #-}

106+compactLE :: (MonadIO m, Unbox a) =>

107+ Int -> Stream m (MutArray a) -> Stream m (MutArray a)

108+-- XXX compactLE can be moved to MutArray/Type if we are not using the parser

109+-- to implement it.

110+compactLE = compactLeAs Unpinned

111+-- The parser version turns out to be a little bit slower.

112+-- compactLE n = Stream.catRights . Stream.parseManyD (pCompactLE n)

113+

114+-- | Pinned version of 'compactLE'.

115+{-# INLINE pinnedCompactLE #-}

116+pinnedCompactLE :: forall m a. (MonadIO m, Unbox a)

117+ => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

118+pinnedCompactLE = compactLeAs Pinned

119+-- pinnedCompactLE n = Stream.catRights . Stream.parseManyD (pPinnedCompactLE n)

120+

121+data SplitState s arr

122+ = Initial s

123+ | Buffering s arr

124+ | Splitting s arr

125+ | Yielding arr (SplitState s arr)

126+ | Finishing

127+

128+-- | Split a stream of arrays on a given separator byte, dropping the separator

129+-- and coalescing all the arrays between two separators into a single array.

130+--

131+{-# INLINE_NORMAL _compactOnByteCustom #-}

132+_compactOnByteCustom

133+ :: MonadIO m

134+ => Word8

135+ -> Stream m (MutArray Word8)

136+ -> Stream m (MutArray Word8)

137+_compactOnByteCustom byte (Stream.Stream step state) =

138+ Stream.Stream step' (Initial state)

139+

140+ where

141+

142+ {-# INLINE_LATE step' #-}

143+ step' gst (Initial st) = do

144+ r <- step gst st

145+ case r of

146+ Stream.Yield arr s -> do

147+ (arr1, marr2) <- breakOn byte arr

148+ return $ case marr2 of

149+ Nothing -> Stream.Skip (Buffering s arr1)

150+ Just arr2 -> Stream.Skip (Yielding arr1 (Splitting s arr2))

151+ Stream.Skip s -> return $ Stream.Skip (Initial s)

152+ Stream.Stop -> return Stream.Stop

153+

154+ step' gst (Buffering st buf) = do

155+ r <- step gst st

156+ case r of

157+ Stream.Yield arr s -> do

158+ (arr1, marr2) <- breakOn byte arr

159+ -- XXX Use spliceExp instead and then rightSize?

160+ buf1 <- splice buf arr1

161+ return $ case marr2 of

162+ Nothing -> Stream.Skip (Buffering s buf1)

163+ Just x -> Stream.Skip (Yielding buf1 (Splitting s x))

164+ Stream.Skip s -> return $ Stream.Skip (Buffering s buf)

165+ Stream.Stop -> return $

166+ if byteLength buf == 0

167+ then Stream.Stop

168+ else Stream.Skip (Yielding buf Finishing)

169+

170+ step' _ (Splitting st buf) = do

171+ (arr1, marr2) <- breakOn byte buf

172+ return $ case marr2 of

173+ Nothing -> Stream.Skip $ Buffering st arr1

174+ Just arr2 -> Stream.Skip $ Yielding arr1 (Splitting st arr2)

175+

176+ step' _ (Yielding arr next) = return $ Stream.Yield arr next

177+ step' _ Finishing = return Stream.Stop

178+

179+-- XXX implement predicate based version of this

180+-- XXX Naming of predicate based vs custom version

181+

182+-- | Split a stream of arrays on a given separator byte, dropping the separator

183+-- and coalescing all the arrays between two separators into a single array.

184+--

185+{-# INLINE compactOnByte #-}

186+compactOnByte

187+ :: (MonadIO m)

188+ => Word8

189+ -> Stream m (MutArray Word8)

190+ -> Stream m (MutArray Word8)

191+-- XXX compare perf of custom vs idiomatic version

192+-- compactOnByte = _compactOnByteCustom

193+-- XXX use spliceExp and rightSize?

194+compactOnByte byte = Stream.splitInnerBy (breakOn byte) splice

195+

196+-- | Like 'compactOnByte' considers the separator in suffix position instead of

197+-- infix position.

198+{-# INLINE compactOnByteSuffix #-}

199+compactOnByteSuffix

200+ :: (MonadIO m)

201+ => Word8

202+ -> Stream m (MutArray Word8)

203+ -> Stream m (MutArray Word8)

204+compactOnByteSuffix byte =

205+ -- XXX use spliceExp and rightSize?

206+ Stream.splitInnerBySuffix

207+ (\arr -> byteLength arr == 0) (breakOn byte) splice

core/src/Streamly/Internal/Data/MutArray/Generic.hs+65−30

@@ -19,14 +19,14 @@ module Streamly.Internal.Data.MutArray.Generic

1919 , nil

2020

2121 -- *** Uninitialized Arrays

22− , new

22+ , emptyOf

2323 -- , newArrayWith

2424

2525 -- *** From streams

26− , writeNUnsafe

27− , writeN

28− , writeWith

29− , write

26+ , unsafeCreateOf

27+ , createOf

28+ , createWith -- createOfMin/createMin/createGE?

29+ , create

3030 , fromStreamN

3131 , fromStream

3232 , fromPureStream

@@ -160,6 +160,13 @@ module Streamly.Internal.Data.MutArray.Generic

160160 -- , appendSliceFrom

161161

162162 , clone

163+

164+ -- * Deprecated

165+ , new

166+ , writeNUnsafe

167+ , writeN

168+ , writeWith

169+ , write

163170 )

164171where

165172

@@ -236,13 +243,13 @@ bottomElement =

236243-- XXX Would be nice if GHC can provide something like newUninitializedArray# so

237244-- that we do not have to write undefined or error in the whole array.

238245

239−-- | @new count@ allocates a zero length array that can be extended to hold

246+-- | @emptyOf count@ allocates a zero length array that can be extended to hold

240247-- up to 'count' items without reallocating.

241248--

242249-- /Pre-release/

243−{-# INLINE new #-}

244−new :: MonadIO m => Int -> m (MutArray a)

245−new n@(I# n#) =

250+{-# INLINE emptyOf #-}

251+emptyOf :: MonadIO m => Int -> m (MutArray a)

252+emptyOf n@(I# n#) =

246253 liftIO

247254 $ IO

248255 $ \s# ->

@@ -251,6 +258,12 @@ new n@(I# n#) =

251258 let ma = MutArray arr# 0 0 n

252259 in (# s1#, ma #)

253260

261+-- XXX Deprecate in major

262+-- {-# DEPRECATED new "Please use emptyOf instead." #-}

263+{-# INLINE new #-}

264+new :: MonadIO m => Int -> m (MutArray a)

265+new = emptyOf

266+

254267-- XXX This could be pure?

255268

256269-- |

@@ -318,7 +331,7 @@ putIndices arr = FL.foldlM' step (return ())

318331

319332 where

320333

321− step () (i, x) = liftIO (putIndex i arr x)

334+ step () (i, x) = putIndex i arr x

322335

323336-- | Modify a given index of an array using a modifier function without checking

324337-- the bounds.

@@ -589,16 +602,16 @@ readRev arr@MutArray{..} =

589602arrayChunkSize :: Int

590603arrayChunkSize = 1024

591604

592−-- | Like 'writeN' but does not check the array bounds when writing. The fold

605+-- | Like 'createOf' but does not check the array bounds when writing. The fold

593606-- driver must not call the step function more than 'n' times otherwise it will

594607-- corrupt the memory and crash. This function exists mainly because any

595608-- conditional in the step function blocks fusion causing 10x performance

596609-- slowdown.

597610--

598611-- /Pre-release/

599−{-# INLINE_NORMAL writeNUnsafe #-}

600−writeNUnsafe :: MonadIO m => Int -> Fold m a (MutArray a)

601−writeNUnsafe n = Fold step initial return return

612+{-# INLINE_NORMAL unsafeCreateOf #-}

613+unsafeCreateOf :: MonadIO m => Int -> Fold m a (MutArray a)

614+unsafeCreateOf n = Fold step initial return return

602615

603616 where

604617

@@ -606,59 +619,81 @@ writeNUnsafe n = Fold step initial return return

606619

607620 step arr x = FL.Partial <$> snocUnsafe arr x

608621

609−-- | @writeN n@ folds a maximum of @n@ elements from the input stream to an

622+{-# DEPRECATED writeNUnsafe "Please use unsafeCreateOf instead." #-}

623+{-# INLINE writeNUnsafe #-}

624+writeNUnsafe :: MonadIO m => Int -> Fold m a (MutArray a)

625+writeNUnsafe = unsafeCreateOf

626+

627+-- | @createOf n@ folds a maximum of @n@ elements from the input stream to an

610628-- 'Array'.

611629--

612−-- >>> writeN n = Fold.take n (MutArray.writeNUnsafe n)

630+-- >>> createOf n = Fold.take n (MutArray.unsafeCreateOf n)

613631--

614632-- /Pre-release/

615−{-# INLINE_NORMAL writeN #-}

633+{-# INLINE_NORMAL createOf #-}

634+createOf :: MonadIO m => Int -> Fold m a (MutArray a)

635+createOf n = FL.take n $ unsafeCreateOf n

636+

637+-- XXX Deprecate in major

638+-- {-# DEPRECATED writeN "Please use createOf instead." #-}

639+{-# INLINE writeN #-}

616640writeN :: MonadIO m => Int -> Fold m a (MutArray a)

617−writeN n = FL.take n $ writeNUnsafe n

641+writeN = createOf

618642

619643-- >>> f n = MutArray.writeAppendWith (* 2) (MutArray.pinnedNew n)

620644-- >>> writeWith n = Fold.rmapM MutArray.rightSize (f n)

621645-- >>> writeWith n = Fold.rmapM MutArray.fromArrayStreamK (MutArray.writeChunks n)

622646

623−-- | @writeWith minCount@ folds the whole input to a single array. The array

647+-- | @createWith minCount@ folds the whole input to a single array. The array

624648-- starts at a size big enough to hold minCount elements, the size is doubled

625649-- every time the array needs to be grown.

626650--

627651-- /Caution! Do not use this on infinite streams./

628652--

629653-- /Pre-release/

630−{-# INLINE_NORMAL writeWith #-}

631−writeWith :: MonadIO m => Int -> Fold m a (MutArray a)

654+{-# INLINE_NORMAL createWith #-}

655+createWith :: MonadIO m => Int -> Fold m a (MutArray a)

632656-- writeWith n = FL.rmapM rightSize $ writeAppendWith (* 2) (pinnedNew n)

633−writeWith elemCount = FL.rmapM extract $ FL.foldlM' step initial

657+createWith elemCount = FL.rmapM extract $ FL.foldlM' step initial

634658

635659 where

636660

637661 initial = do

638− when (elemCount < 0) $ error "writeWith: elemCount is negative"

639− liftIO $ new elemCount

662+ when (elemCount < 0) $ error "createWith: elemCount is negative"

663+ new elemCount

640664

641665 step arr@(MutArray _ start end bound) x

642666 | end == bound = do

643667 let oldSize = end - start

644668 newSize = max (oldSize * 2) 1

645− arr1 <- liftIO $ realloc newSize arr

669+ arr1 <- realloc newSize arr

646670 snocUnsafe arr1 x

647671 step arr x = snocUnsafe arr x

648672

649− -- extract = liftIO . rightSize

673+ -- extract = rightSize

650674 extract = return

651675

676+{-# DEPRECATED writeWith "Please use createWith instead." #-}

677+{-# INLINE writeWith #-}

678+writeWith :: MonadIO m => Int -> Fold m a (MutArray a)

679+writeWith = createWith

680+

652681-- | Fold the whole input to a single array.

653682--

654−-- Same as 'writeWith' using an initial array size of 'arrayChunkSize' bytes

683+-- Same as 'createWith' using an initial array size of 'arrayChunkSize' bytes

655684-- rounded up to the element size.

656685--

657686-- /Caution! Do not use this on infinite streams./

658687--

688+{-# INLINE create #-}

689+create :: MonadIO m => Fold m a (MutArray a)

690+create = writeWith arrayChunkSize

691+

692+-- XXX Deprecate in major

693+-- {-# DEPRECATED write "Please use create instead." #-}

659694{-# INLINE write #-}

660695write :: MonadIO m => Fold m a (MutArray a)

661−write = writeWith arrayChunkSize

696+write = create

662697

663698-- | Create a 'MutArray' from the first @n@ elements of a stream. The

664699-- array is allocated to size @n@, if the stream terminates before @n@

@@ -683,7 +718,7 @@ fromList xs = fromStream $ D.fromList xs

683718{-# INLINABLE fromPureStream #-}

684719fromPureStream :: MonadIO m => Stream Identity a -> m (MutArray a)

685720fromPureStream xs =

686− liftIO $ D.fold write $ D.morphInner (return . runIdentity) xs

721+ D.fold write $ D.morphInner (return . runIdentity) xs

687722

688723-------------------------------------------------------------------------------

689724-- Chunking

@@ -815,7 +850,7 @@ putSliceUnsafe src srcStart dst dstStart len = liftIO $ do

815850

816851{-# INLINE clone #-}

817852clone :: MonadIO m => MutArray a -> m (MutArray a)

818−clone src = liftIO $ do

853+clone src = do

819854 let len = arrLen src

820855 dst <- new len

821856 putSliceUnsafe src 0 dst 0 len

core/src/Streamly/Internal/Data/MutArray/Stream.hsnew file+145

@@ -0,0 +1,145 @@

1+-- |

2+-- Module : Streamly.Internal.Data.MutArray.Stream

4+-- License : BSD3-3-Clause

5+-- Maintainer : streamly@composewell.com

6+-- Stability : experimental

7+-- Portability : GHC

8+--

9+-- Combinators to efficiently manipulate streams of mutable arrays.

10+--

11+-- We can either push these in the MutArray module with a "chunks" prefix or

12+-- keep this as a separate module and release it.

13+--

14+module Streamly.Internal.Data.MutArray.Stream

15+{-# DEPRECATED "Please use \"Streamly.Internal.Data.MutArray\" instead." #-}

16+ (

17+ -- * Generation

18+ MArray.chunksOf

19+ , MArray.pinnedChunksOf

20+ , MArray.writeChunks -- chunksWrite?

21+ , MArray.splitOn -- chunksSplitOn

22+

23+ -- * Compaction

24+ , packArraysChunksOf

25+ , MArray.SpliceState (..)

26+ , lpackArraysChunksOf

27+ , compact -- chunksCompact

28+ , compactLE

29+ , compactEQ

30+ , compactGE

31+

32+ -- * Elimination

33+ , MArray.flattenArrays -- chunksConcat

34+ , MArray.flattenArraysRev -- chunksConcatRev

35+ , MArray.fromArrayStreamK -- chunksCoalesce

36+ )

37+where

38+

39+import Control.Monad.IO.Class (MonadIO(..))

40+import Streamly.Internal.Data.Unbox (Unbox)

41+import Streamly.Internal.Data.MutArray.Type (MutArray(..))

42+import Streamly.Internal.Data.Fold.Type (Fold(..))

43+import Streamly.Internal.Data.Parser (ParseError)

44+import Streamly.Internal.Data.Stream.Type (Stream)

45+

46+import qualified Streamly.Internal.Data.MutArray as MArray

47+import qualified Streamly.Internal.Data.Fold.Type as FL

48+import qualified Streamly.Internal.Data.Parser as ParserD

49+import qualified Streamly.Internal.Data.Stream as D

50+

51+-------------------------------------------------------------------------------

52+-- Compact

53+-------------------------------------------------------------------------------

54+

55+-- XXX This can be removed once compactLEFold/compactLE are implemented.

56+--

57+-- | This mutates the first array (if it has space) to append values from the

58+-- second one. This would work for immutable arrays as well because an

59+-- immutable array never has space so a new array is allocated instead of

60+-- mutating it.

61+--

62+-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

63+-- maximum specified size. Note that if a single array is bigger than the

64+-- specified size we do not split it to fit. When we coalesce multiple arrays

65+-- if the size would exceed the specified size we do not coalesce therefore the

66+-- actual array size may be less than the specified chunk size.

67+--

68+-- @since 0.7.0

69+{-# INLINE packArraysChunksOf #-}

70+packArraysChunksOf :: (MonadIO m, Unbox a)

71+ => Int -> D.Stream m (MutArray a) -> D.Stream m (MutArray a)

72+packArraysChunksOf = MArray.compactLE

73+

74+-- XXX Remove this once compactLEFold is implemented

75+-- lpackArraysChunksOf = Fold.many compactLEFold

76+--

77+{-# INLINE lpackArraysChunksOf #-}

78+lpackArraysChunksOf :: (MonadIO m, Unbox a)

79+ => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()

80+lpackArraysChunksOf = MArray.lCompactGE

81+

82+-- XXX Same as compactLE, to be removed once that is implemented.

83+--

84+-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

85+-- maximum specified size in bytes.

86+--

87+-- /Internal/

88+{-# INLINE compact #-}

89+compact :: (MonadIO m, Unbox a)

90+ => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

91+compact = packArraysChunksOf

92+

93+-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

94+-- maximum specified size. Note that if a single array is bigger than the

95+-- specified size we do not split it to fit. When we coalesce multiple arrays

96+-- if the size would exceed the specified size we do not coalesce therefore the

97+-- actual array size may be less than the specified chunk size.

98+--

99+-- /Internal/

100+{-# INLINE compactLEParserD #-}

101+compactLEParserD ::

102+ forall m a. (MonadIO m, Unbox a)

103+ => Int -> ParserD.Parser (MutArray a) m (MutArray a)

104+compactLEParserD = MArray.pCompactLE

105+

106+-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

107+-- minimum specified size. Note that if all the arrays in the stream together

108+-- are smaller than the specified size the resulting array will be smaller than

109+-- the specified size. When we coalesce multiple arrays if the size would exceed

110+-- the specified size we stop coalescing further.

111+--

112+-- /Internal/

113+{-# INLINE compactGEFold #-}

114+compactGEFold ::

115+ forall m a. (MonadIO m, Unbox a)

116+ => Int -> FL.Fold m (MutArray a) (MutArray a)

117+compactGEFold = MArray.fCompactGE

118+

119+-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

120+-- maximum specified size in bytes.

121+--

122+-- /Internal/

123+compactLE :: (MonadIO m, Unbox a) =>

124+ Int -> Stream m (MutArray a) -> Stream m (Either ParseError (MutArray a))

125+compactLE n = D.parseManyD (compactLEParserD n)

126+

127+-- | Like 'compactLE' but generates arrays of exactly equal to the size

128+-- specified except for the last array in the stream which could be shorter.

129+--

130+-- /Unimplemented/

131+{-# INLINE compactEQ #-}

132+compactEQ :: -- (MonadIO m, Unbox a) =>

133+ Int -> Stream m (MutArray a) -> Stream m (MutArray a)

134+compactEQ _n _xs = undefined

135+ -- IsStream.fromStreamD $ D.foldMany (compactEQFold n) (IsStream.toStreamD xs)

136+

137+-- | Like 'compactLE' but generates arrays of size greater than or equal to the

138+-- specified except for the last array in the stream which could be shorter.

139+--

140+-- /Internal/

141+{-# INLINE compactGE #-}

142+compactGE ::

143+ (MonadIO m, Unbox a)

144+ => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

145+compactGE n = D.foldMany (compactGEFold n)

core/src/Streamly/Internal/Data/MutArray/Type.hs+1271−417

@@ -28,57 +28,80 @@

2828

2929module Streamly.Internal.Data.MutArray.Type

3030 (

31− -- * Type

31+ -- ** Type

3232 -- $arrayNotes

3333 MutArray (..)

34− , MutableByteArray

35− , touch

34+

35+ -- ** Conversion

36+ -- *** Pinned and Unpinned

3637 , pin

3738 , unpin

3839 , isPinned

3940

40− -- * Constructing and Writing

41+ -- ** Casting

42+ , cast

43+ , castUnsafe -- XXX unsafeCast

44+ , asBytes

45+ , unsafePinnedAsPtr

46+ , unsafeAsPtr

47+

4148 -- ** Construction

42− , nil

49+ , empty

50+

51+ -- *** New

52+ -- | New arrays are always empty arrays with some reserve capacity to

53+ -- extend the length without reallocating.

54+ , emptyOf

55+ , newArrayWith -- emptyAlignedWith

56+ , pinnedEmptyOf

57+ , pinnedNewAligned -- XXX not required

58+ -- , new -- uninitialized array of specified length

59+

60+ -- *** Cloning

61+ , clone

62+ , pinnedClone

4363

44− -- *** Uninitialized Arrays

45− , pinnedNew

46− , pinnedNewBytes

47− , pinnedNewAligned

48− , new

49− , newArrayWith

64+ -- *** Slicing

65+ -- | Get a subarray without copying

66+ , getSliceUnsafe -- XXX unsafeGetSlice

67+ , getSlice

68+ , splitAt -- XXX should be able to express using getSlice

69+ , breakOn

5070

51− -- *** From streams

71+ -- *** Stream Folds

5272 , ArrayUnsafe (..)

53− , writeNWithUnsafe

54− , writeNWith

55− , writeNUnsafe

56− , pinnedWriteNUnsafe

57− , writeN

58− , pinnedWriteN

59− , pinnedWriteNAligned

60−

61− , writeWith

62− , write

63− , pinnedWrite

64−

65− , writeRevN

66− -- , writeRev

67−

68− -- ** From containers

73+ , unsafeCreateOfWith

74+ , unsafeCreateOf

75+ , unsafePinnedCreateOf

76+ , pinnedCreateOf

77+ , createOfWith

78+ , createOf

79+ , revCreateOf

80+

81+ , pinnedCreate

82+ , createWith

83+ , create

84+ -- , revCreate

85+

86+ -- *** From containers

6987 , fromListN

7088 , pinnedFromListN

7189 , fromList

7290 , pinnedFromList

7391 , fromListRevN

7492 , fromListRev

75− , fromStreamDN

76− , fromStreamD

93+ , fromStreamN

94+ , fromStream

95+ , fromPureStreamN

7796 , fromPureStream

97+ , fromByteStr#

98+ , fromPtrN

99+ , fromChunksK

100+ , fromChunksRealloced -- fromSmallChunks

78101

79− -- * Random writes

102+ -- ** Random writes

80103 , putIndex

81− , putIndexUnsafe

104+ , putIndexUnsafe -- XXX unsafePutIndex

82105 , putIndices

83106 -- , putFromThenTo

84107 -- , putFrom -- start writing at the given position

@@ -86,78 +109,71 @@ module Streamly.Internal.Data.MutArray.Type

86109 -- , putFromTo

87110 -- , putFromRev

88111 -- , putUptoRev

89− , modifyIndexUnsafe

112+ , modifyIndexUnsafe -- XXX unsafeModifyIndex

90113 , modifyIndex

91114 , modifyIndices

92115 , modify

93116 , swapIndices

94117 , unsafeSwapIndices

95118

96− -- * Growing and Shrinking

97− -- Arrays grow only at the end, though it is possible to grow on both sides

98− -- and therefore have a cons as well as snoc. But that will require two

99− -- bounds in the array representation.

100−

101− -- ** Appending elements

102− , snocWith

103− , snoc

104− , snocLinear

105− , snocMay

106− , snocUnsafe

107−

108− -- ** Appending streams

109− , writeAppendNUnsafe

110− , writeAppendN

111− , writeAppendWith

112− , writeAppend

119+ -- ** Reading

113120

114− -- * Eliminating and Reading

115−

116− -- ** To streams

117− , reader

118− , readerRevWith

119− , readerRev

121+ -- *** Indexing

122+ , getIndex

123+ , getIndexUnsafe -- XXX unsafeGetIndex

124+ -- , getFromThenTo

125+ , getIndexRev -- getRevIndex?

126+ , indexReader

127+ , indexReaderWith

120128

121− -- ** To containers

122− , toStreamDWith

123− , toStreamDRevWith

124− , toStreamKWith

125− , toStreamKRevWith

129+ -- *** To Streams

126130 , read

127131 , readRev

132+ , toStreamWith

133+ , toStreamRevWith

128134 , toStreamK

135+ , toStreamKWith

129136 , toStreamKRev

137+ , toStreamKRevWith

138+

139+ -- *** To Containers

130140 , toList

131141

142+ -- *** Unfolds

132143 -- experimental

133144 , producerWith

134145 , producer

135146

136− -- ** Random reads

137− , getIndex

138− , getIndexUnsafe

139− , getIndices

140− , getIndicesD

141− -- , getFromThenTo

142− , getIndexRev

143−

144− -- * Memory Management

145− , blockSize

146− , arrayChunkBytes

147− , allocBytesToElemCount

148− , realloc

149− , resize

150− , resizeExp

151− , rightSize

147+ , reader

148+ , readerRevWith

149+ , readerRev

152150

153− -- * Size

151+ -- ** Size and Capacity

152+ -- *** Size

154153 , length

155154 , byteLength

155+

156+ -- *** Capacity

156157 -- , capacity

157158 , byteCapacity

158159 , bytesFree

159160

160− -- * In-place Mutation Algorithms

161+ -- *** Capacity Management

162+ , blockSize

163+ , arrayChunkBytes

164+ , allocBytesToElemCount

165+ , realloc -- this is "resize"

166+ , grow

167+ , growExp

168+ , rightSize

169+

170+ -- ** Folding

171+ , foldl'

172+ , foldr

173+ , byteCmp

174+ , byteEq

175+

176+ -- ** In-place Mutation Algorithms

161177 , strip

162178 , reverse

163179 , permute

@@ -167,62 +183,121 @@ module Streamly.Internal.Data.MutArray.Type

167183 , mergeBy

168184 , bubble

169185

170− -- * Casting

171− , cast

172− , castUnsafe

173− , asBytes

174− , asPtrUnsafe

175−

176− -- * Folding

177− , foldl'

178− , foldr

179− , cmp

180−

181− -- * Arrays of arrays

182− -- We can add dimensionality parameter to the array type to get

183− -- multidimensional arrays. Multidimensional arrays would just be a

184− -- convenience wrapper on top of single dimensional arrays.

186+ -- ** Growing and Shrinking

187+ -- | Arrays grow only at the end, though it is possible to grow on both

188+ -- sides and therefore have a cons as well as snoc. But that will require

189+ -- both lower and upper bound in the array representation.

185190

186− -- | Operations dealing with multiple arrays, streams of arrays or

187− -- multidimensional array representations.

188−

189− -- ** Construct from streams

190− , chunksOf

191− , pinnedChunksOf

192− , writeChunks

193−

194− -- ** Eliminate to streams

195− , flattenArrays

196− , flattenArraysRev

197− , fromArrayStreamK

198−

199− -- ** Construct from arrays

200− -- get chunks without copying

201− , getSliceUnsafe

202− , getSlice

203− -- , getSlicesFromLenN

204− , splitAt -- XXX should be able to express using getSlice

205− , breakOn

191+ -- *** Appending elements

192+ , snocWith

193+ , snoc

194+ , snocLinear

195+ , snocMay

196+ , snocUnsafe -- XXX unsafeSnoc

206197

207− -- ** Cloning arrays

208− , clone

209− , pinnedClone

198+ -- *** Appending streams

199+ , unsafeAppendN

200+ , appendN

201+ , appendWith

202+ , append

210203

211− -- ** Appending arrays

204+ -- *** Appending arrays

212205 , spliceCopy

213206 , spliceWith

214207 , splice

215208 , spliceExp

216− , spliceUnsafe

209+ , spliceUnsafe -- XXX unsafeSplice

217210 -- , putSlice

218211 -- , appendSlice

219212 -- , appendSliceFrom

220213

221− -- * Utilities

214+ -- ** Serialization using Unbox

215+ , pokeAppend

216+ , pokeAppendMay

217+ , pokeSkipUnsafe -- unsafePokeSkip

218+

219+ -- ** Deserialization using Unbox

220+ , peekUncons

221+ , peekUnconsUnsafe -- unsafePeekUncons

222+ , peekSkipUnsafe -- unsafePeekSkip

223+

224+ -- Arrays of arrays

225+ -- We can add dimensionality parameter to the array type to get

226+ -- multidimensional arrays. Multidimensional arrays would just be a

227+ -- convenience wrapper on top of single dimensional arrays.

228+

229+ -- ** Streams of Arrays

230+ -- *** Chunk

231+ -- | Group a stream into arrays.

232+ , chunksOf

233+ , pinnedChunksOf

234+ , buildChunks

235+

236+ -- *** Split

237+ -- | Split an array into slices.

238+

239+ -- , getSlicesFromLenN

240+ , splitOn

241+ -- , slicesOf

242+

243+ -- *** Concat

244+ -- | Append the arrays in a stream to form a stream of elements.

245+ , concatWith

246+ , concatRevWith

247+ , concat

248+ , concatRev

249+

250+ -- *** Compact

251+ -- | Append the arrays in a stream to form a stream of larger arrays.

252+ , SpliceState (..)

253+ , pCompactLE

254+ , pPinnedCompactLE

255+ , compactLeAs

256+ , fCompactGE

257+ , fPinnedCompactGE

258+ , lCompactGE

259+ , lPinnedCompactGE

260+ , compactGE

261+ , compactEQ

262+

263+ -- ** Utilities

222264 , roundUpToPower2

223265 , memcpy

224266 , memcmp

225267 , c_memchr

268+

269+ -- * Deprecated

270+ , asPtrUnsafe

271+ , writeChunks

272+ , flattenArrays

273+ , flattenArraysRev

274+ , fromArrayStreamK

275+ , fromStreamDN

276+ , fromStreamD

277+ , cmp

278+ , getIndices

279+ , getIndicesWith

280+ , resize

281+ , resizeExp

282+ , nil

283+ , new

284+ , pinnedNew

285+ , pinnedNewBytes

286+ , writeAppendNUnsafe

287+ , writeAppendN

288+ , writeAppendWith

289+ , writeAppend

290+ , writeNWithUnsafe

291+ , writeNWith

292+ , writeNUnsafe

293+ , pinnedWriteNUnsafe

294+ , writeN

295+ , pinnedWriteN

296+ , pinnedWriteNAligned -- XXX not required

297+ , writeWith

298+ , write

299+ , pinnedWrite

300+ , writeRevN

226301 )

227302where

228303

@@ -233,49 +308,52 @@ where

233308

234309import Control.Monad (when, void)

235310import Control.Monad.IO.Class (MonadIO(..))

311+import Data.Bifunctor (first)

236312import Data.Bits (shiftR, (.|.), (.&.))

237313import Data.Functor.Identity (Identity(..))

238314import Data.Proxy (Proxy(..))

239315import Data.Word (Word8)

240316import Foreign.C.Types (CSize(..), CInt(..))

241317import Foreign.Ptr (plusPtr, minusPtr, nullPtr)

242−import Streamly.Internal.Data.Unbox

243− ( MutableByteArray(..)

244− , Unbox(..)

318+import Streamly.Internal.Data.MutByteArray.Type

319+ ( MutByteArray(..)

245320 , PinnedState(..)

246321 , getMutableByteArray#

247− , touch

248322 , putSliceUnsafe

249323 )

324+import Streamly.Internal.Data.Unbox (Unbox(..))

250325import GHC.Base

251326 ( IO(..)

252327 , Int(..)

253− , byteArrayContents#

254328 , compareByteArrays#

255329 , copyMutableByteArray#

256330 )

257331import GHC.Base (noinline)

258−import GHC.Exts (unsafeCoerce#)

332+import GHC.Exts (unsafeCoerce#, Addr#)

259333import GHC.Ptr (Ptr(..))

260334

261335import Streamly.Internal.Data.Fold.Type (Fold(..))

262336import Streamly.Internal.Data.Producer.Type (Producer (..))

263337import Streamly.Internal.Data.Stream.Type (Stream)

338+import Streamly.Internal.Data.Parser.Type (Parser (..))

264339import Streamly.Internal.Data.StreamK.Type (StreamK)

265340import Streamly.Internal.Data.SVar.Type (adaptState, defState)

341+import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))

266342import Streamly.Internal.Data.Unfold.Type (Unfold(..))

267343import Streamly.Internal.System.IO (arrayPayloadSize, defaultChunkSize)

268344

269345import qualified Streamly.Internal.Data.Fold.Type as FL

346+import qualified Streamly.Internal.Data.MutByteArray.Type as Unboxed

347+import qualified Streamly.Internal.Data.Parser.Type as Parser

348+-- import qualified Streamly.Internal.Data.Fold.Type as Fold

270349import qualified Streamly.Internal.Data.Producer as Producer

271350import qualified Streamly.Internal.Data.Stream.Type as D

272351import qualified Streamly.Internal.Data.Stream.Lift as D

273352import qualified Streamly.Internal.Data.StreamK.Type as K

274−import qualified Streamly.Internal.Data.Unbox as Unboxed

275353import qualified Prelude

276354

277355import Prelude hiding

278− (Foldable(..), read, unlines, splitAt, reverse, truncate)

356+ (Foldable(..), concat, read, unlines, splitAt, reverse, truncate)

279357

280358#include "DocTestDataMutArray.hs"

281359

@@ -283,6 +361,7 @@ import Prelude hiding

283361-- Foreign helpers

284362-------------------------------------------------------------------------------

285363

364+-- NOTE: Have to be "ccall unsafe" so that we can pass unpinned memory to these

286365foreign import ccall unsafe "string.h memcpy" c_memcpy

287366 :: Ptr Word8 -> Ptr Word8 -> CSize -> IO (Ptr Word8)

288367

@@ -292,6 +371,9 @@ foreign import ccall unsafe "string.h memchr" c_memchr

292371foreign import ccall unsafe "string.h memcmp" c_memcmp

293372 :: Ptr Word8 -> Ptr Word8 -> CSize -> IO CInt

294373

374+foreign import ccall unsafe "string.h strlen" c_strlen

375+ :: Ptr Word8 -> IO CSize

376+

295377-- | Given an 'Unboxed' type (unused first arg) and a number of bytes, return

296378-- how many elements of that type will completely fit in those bytes.

297379--

@@ -346,7 +428,7 @@ data MutArray a =

346428 -- The array is a range into arrContents. arrContents may be a superset of

347429 -- the slice represented by the array. All offsets are in bytes.

348430 MutArray

349− { arrContents :: {-# UNPACK #-} !MutableByteArray

431+ { arrContents :: {-# UNPACK #-} !MutByteArray

350432 , arrStart :: {-# UNPACK #-} !Int -- ^ index into arrContents

351433 , arrEnd :: {-# UNPACK #-} !Int -- ^ index into arrContents

352434 -- Represents the first invalid index of

@@ -358,8 +440,8 @@ data MutArray a =

358440-- Pinning & Unpinning

359441-------------------------------------------------------------------------------

360442

361−-- | Copy the 'MutArray' to pinned memory if unpinned, else do nothing. The

362−-- overhead is a copy if the input array is unpinned.

443+-- | Return a copy of the array in pinned memory if unpinned, else return the

444+-- original array.

363445{-# INLINE pin #-}

364446pin :: MutArray a -> IO (MutArray a)

365447pin arr@MutArray{..} =

@@ -367,8 +449,8 @@ pin arr@MutArray{..} =

367449 then pure arr

368450 else pinnedClone arr

369451

370−-- | Copy the 'MutArray' to unpinned memory if pinned, else do nothing. The

371−-- overhead is a copy if the input array is pinned.

452+-- | Return a copy of the array in unpinned memory if pinned, else return the

453+-- original array.

372454{-# INLINE unpin #-}

373455unpin :: MutArray a -> IO (MutArray a)

374456unpin arr@MutArray{..} =

@@ -376,6 +458,7 @@ unpin arr@MutArray{..} =

376458 then clone arr

377459 else pure arr

378460

461+-- | Return 'True' if the array is allocated in pinned memory.

379462{-# INLINE isPinned #-}

380463isPinned :: MutArray a -> Bool

381464isPinned MutArray{..} = Unboxed.isPinned arrContents

@@ -396,6 +479,8 @@ isPinned MutArray{..} = Unboxed.isPinned arrContents

396479-- we need more than that. See stg_pinnedNewAlignedByteArrayzh and

397480-- allocatePinned in GHC source.

398481

482+-- XXX Rename to emptyAlignedWith, alignSize should be first arg.

483+

399484-- | @newArrayWith allocator alignment count@ allocates a new array of zero

400485-- length and with a capacity to hold @count@ elements, using @allocator

401486-- size alignment@ as the memory allocator function.

@@ -408,7 +493,7 @@ isPinned MutArray{..} = Unboxed.isPinned arrContents

408493-- /Pre-release/

409494{-# INLINE newArrayWith #-}

410495newArrayWith :: forall m a. (MonadIO m, Unbox a)

411− => (Int -> Int -> m MutableByteArray) -> Int -> Int -> m (MutArray a)

496+ => (Int -> Int -> m MutByteArray) -> Int -> Int -> m (MutArray a)

412497newArrayWith alloc alignSize count = do

413498 let size = max (count * SIZE_OF(a)) 0

414499 contents <- alloc size alignSize

@@ -419,12 +504,25 @@ newArrayWith alloc alignSize count = do

419504 , arrBound = size

420505 }

421506

507+-- For arrays "nil" sounds a bit odd. empty is better. The only problem with

508+-- empty is that it is also used by the Alternative type class. But assuming we

509+-- will mostly import the Array module qualified this should be fine.

510+

511+-- | Create an empty array.

512+empty ::

513+#ifdef DEVBUILD

514+ Unbox a =>

515+#endif

516+ MutArray a

517+empty = MutArray Unboxed.empty 0 0 0

518+

519+{-# DEPRECATED nil "Please use empty instead." #-}

422520nil ::

423521#ifdef DEVBUILD

424522 Unbox a =>

425523#endif

426524 MutArray a

427−nil = MutArray Unboxed.nil 0 0 0

525+nil = empty

428526

429527{-# INLINE newBytesAs #-}

430528newBytesAs :: MonadIO m =>

@@ -441,12 +539,13 @@ newBytesAs ps bytes = do

441539 , arrBound = bytes

442540 }

443541

444−-- | Allocates a pinned empty array that can hold 'count' items. The memory of

445−-- the array is uninitialized and the allocation is aligned as per the

446−-- 'Unboxed' instance of the type.

542+-- | Allocates a pinned empty array that with a reserved capacity of bytes.

543+-- The memory of the array is uninitialized and the allocation is aligned as

544+-- per the 'Unboxed' instance of the type.

447545--

448546-- /Pre-release/

449547{-# INLINE pinnedNewBytes #-}

548+{-# DEPRECATED pinnedNewBytes "Please use pinnedEmptyOf with appropriate calculation" #-}

450549pinnedNewBytes :: MonadIO m =>

451550#ifdef DEVBUILD

452551 Unbox a =>

@@ -471,21 +570,31 @@ newAs ps =

471570 (error "new: alignment is not used in unpinned arrays.")

472571

473572-- XXX can unaligned allocation be more efficient when alignment is not needed?

474−--

475−-- | Allocates an empty pinned array that can hold 'count' items. The memory of

476−-- the array is uninitialized and the allocation is aligned as per the 'Unboxed'

477−-- instance of the type.

478−--

573+

574+-- | Allocates a pinned array of zero length but growable to the specified

575+-- capacity without reallocation.

576+{-# INLINE pinnedEmptyOf #-}

577+pinnedEmptyOf :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a)

578+pinnedEmptyOf = newAs Pinned

579+

580+-- XXX Deprecate in major

581+-- {-# DEPRECATED pinnedNew "Please use pinnedEmptyOf instead." #-}

479582{-# INLINE pinnedNew #-}

480583pinnedNew :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a)

481−pinnedNew = newAs Pinned

584+pinnedNew = pinnedEmptyOf

482585

483−-- | Allocates an empty unpinned array that can hold 'count' items. The memory

484−-- of the array is uninitialized.

586+-- | Allocates an unpinned array of zero length but growable to the specified

587+-- capacity without reallocation.

485588--

589+{-# INLINE emptyOf #-}

590+emptyOf :: (MonadIO m, Unbox a) => Int -> m (MutArray a)

591+emptyOf = newAs Unpinned

592+

593+-- XXX Deprecate in major

594+-- {-# DEPRECATED new "Please use emptyOf instead." #-}

486595{-# INLINE new #-}

487596new :: (MonadIO m, Unbox a) => Int -> m (MutArray a)

488−new = newAs Unpinned

597+new = emptyOf

489598

490599-------------------------------------------------------------------------------

491600-- Random writes

@@ -501,7 +610,7 @@ putIndexUnsafe :: forall m a. (MonadIO m, Unbox a)

501610putIndexUnsafe i MutArray{..} x = do

502611 let index = INDEX_OF(arrStart, i, a)

503612 assert (i >= 0 && INDEX_VALID(index, arrEnd, a)) (return ())

504− liftIO $ pokeByteIndex index arrContents x

613+ liftIO $ pokeAt index arrContents x

505614

506615invalidIndex :: String -> Int -> a

507616invalidIndex label i =

@@ -519,7 +628,7 @@ putIndex :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> a -> m ()

519628putIndex i MutArray{..} x = do

520629 let index = INDEX_OF(arrStart,i,a)

521630 if i >= 0 && INDEX_VALID(index,arrEnd,a)

522− then liftIO $ pokeByteIndex index arrContents x

631+ then liftIO $ pokeAt index arrContents x

523632 else invalidIndex "putIndex" i

524633

525634-- | Write an input stream of (index, value) pairs to an array. Throws an

@@ -533,7 +642,7 @@ putIndices arr = FL.foldlM' step (return ())

533642

534643 where

535644

536− step () (i, x) = liftIO (putIndex i arr x)

645+ step () (i, x) = putIndex i arr x

537646

538647-- | Modify a given index of an array using a modifier function.

539648--

@@ -545,9 +654,9 @@ modifyIndexUnsafe :: forall m a b. (MonadIO m, Unbox a) =>

545654modifyIndexUnsafe i MutArray{..} f = liftIO $ do

546655 let index = INDEX_OF(arrStart,i,a)

547656 assert (i >= 0 && INDEX_NEXT(index,a) <= arrEnd) (return ())

548− r <- peekByteIndex index arrContents

657+ r <- peekAt index arrContents

549658 let (x, res) = f r

550− pokeByteIndex index arrContents x

659+ pokeAt index arrContents x

551660 return res

552661

553662-- | Modify a given index of an array using a modifier function.

@@ -559,9 +668,9 @@ modifyIndex i MutArray{..} f = do

559668 let index = INDEX_OF(arrStart,i,a)

560669 if i >= 0 && INDEX_VALID(index,arrEnd,a)

561670 then liftIO $ do

562− r <- peekByteIndex index arrContents

671+ r <- peekAt index arrContents

563672 let (x, res) = f r

564− pokeByteIndex index arrContents x

673+ pokeAt index arrContents x

565674 return res

566675 else invalidIndex "modifyIndex" i

567676

@@ -595,8 +704,8 @@ modify MutArray{..} f = liftIO $

595704

596705 go i =

597706 when (INDEX_VALID(i,arrEnd,a)) $ do

598− r <- peekByteIndex i arrContents

599− pokeByteIndex i arrContents (f r)

707+ r <- peekAt i arrContents

708+ pokeAt i arrContents (f r)

600709 go (INDEX_NEXT(i,a))

601710

602711-- XXX We could specify the number of bytes to swap instead of Proxy. Need

@@ -605,15 +714,15 @@ modify MutArray{..} f = liftIO $

605714swapArrayByteIndices ::

606715 forall a. Unbox a

607716 => Proxy a

608− -> MutableByteArray

717+ -> MutByteArray

609718 -> Int

610719 -> Int

611720 -> IO ()

612721swapArrayByteIndices _ arrContents i1 i2 = do

613− r1 <- peekByteIndex i1 arrContents

614− r2 <- peekByteIndex i2 arrContents

615− pokeByteIndex i1 arrContents (r2 :: a)

616− pokeByteIndex i2 arrContents (r1 :: a)

722+ r1 <- peekAt i1 arrContents

723+ r2 <- peekAt i2 arrContents

724+ pokeAt i1 arrContents (r2 :: a)

725+ pokeAt i2 arrContents (r1 :: a)

617726

618727-- | Swap the elements at two indices without validating the indices.

619728--

@@ -729,26 +838,23 @@ arrayChunkBytes = 1024

729838roundDownTo :: Int -> Int -> Int

730839roundDownTo elemSize size = size - (size `mod` elemSize)

731840

732−-- XXX See if resizing can be implemented by reading the old array as a stream

733−-- and then using writeN to the new array.

734−--

735841-- NOTE: we are passing elemSize explicitly to avoid an Unboxed constraint.

736842-- Since this is not inlined Unboxed consrraint leads to dictionary passing

737843-- which complicates some inspection tests.

738844--

739−{-# NOINLINE reallocExplicit #-}

740−reallocExplicit :: Int -> Int -> MutArray a -> IO (MutArray a)

741−reallocExplicit elemSize newCapacityInBytes MutArray{..} = do

845+{-# NOINLINE reallocExplicitAs #-}

846+reallocExplicitAs :: PinnedState -> Int -> Int -> MutArray a -> IO (MutArray a)

847+reallocExplicitAs ps elemSize newCapacityInBytes MutArray{..} = do

742848 assertM(arrEnd <= arrBound)

743849

744850 -- Allocate new array

745851 let newCapMaxInBytes = roundUpLargeArray newCapacityInBytes

746852 contents <-

747− if Unboxed.isPinned arrContents

748− then Unboxed.pinnedNewBytes newCapMaxInBytes

749− else Unboxed.newBytes newCapMaxInBytes

750− let !(MutableByteArray mbarrFrom#) = arrContents

751− !(MutableByteArray mbarrTo#) = contents

853+ if ps == Pinned

854+ then Unboxed.pinnedNew newCapMaxInBytes

855+ else Unboxed.new newCapMaxInBytes

856+ let !(MutByteArray mbarrFrom#) = arrContents

857+ !(MutByteArray mbarrTo#) = contents

752858

753859 -- Copy old data

754860 let oldStart = arrStart

@@ -769,6 +875,10 @@ reallocExplicit elemSize newCapacityInBytes MutArray{..} = do

769875 , arrBound = newCapInBytes

770876 }

771877

878+-- XXX Should these be called reallocBytes etc?

879+-- XXX We may also need reallocAs to allocate as pinned/unpinned explicitly. In

880+-- fact clone/pinnedClone can be implemented using reallocAs.

881+

772882-- | @realloc newCapacity array@ reallocates the array to the specified

773883-- capacity in bytes.

774884--

@@ -780,7 +890,12 @@ reallocExplicit elemSize newCapacityInBytes MutArray{..} = do

780890-- If the original array is pinned, the newly allocated array is also pinned.

781891{-# INLINABLE realloc #-}

782892realloc :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)

783−realloc bytes arr = liftIO $ reallocExplicit (SIZE_OF(a)) bytes arr

893+realloc bytes arr =

894+ let ps =

895+ if isPinned arr

896+ then Pinned

897+ else Unpinned

898+ in liftIO $ reallocExplicitAs ps (SIZE_OF(a)) bytes arr

784899

785900-- | @reallocWith label capSizer minIncrBytes array@. The label is used

786901-- in error messages and the capSizer is used to determine the capacity of the

@@ -803,14 +918,14 @@ reallocWith label capSizer minIncrBytes arr = do

803918 where

804919

805920 badSize newSize =

806− concat

921+ Prelude.concat

807922 [ label

808923 , ": new array size (in bytes) is less than required size "

809924 , show newSize

810925 , ". Please check the sizing function passed."

811926 ]

812927

813−-- | @resize newCapacity array@ changes the total capacity of the array so that

928+-- | @grow newCapacity array@ changes the total capacity of the array so that

814929-- it is enough to hold the specified number of elements. Nothing is done if

815930-- the specified capacity is less than the length of the array.

816931--

@@ -818,34 +933,46 @@ reallocWith label capSizer minIncrBytes arr = do

818933-- the block size (4K).

819934--

820935-- /Pre-release/

821−{-# INLINE resize #-}

822−resize :: forall m a. (MonadIO m, Unbox a) =>

936+{-# INLINE grow #-}

937+grow :: forall m a. (MonadIO m, Unbox a) =>

823938 Int -> MutArray a -> m (MutArray a)

824−resize nElems arr@MutArray{..} = do

939+grow nElems arr@MutArray{..} = do

825940 let req = SIZE_OF(a) * nElems

826− len = arrEnd - arrStart

827− if req < len

941+ cap = arrBound - arrStart

942+ if req < cap

828943 then return arr

829944 else realloc req arr

830945

831−-- | Like 'resize' but if the byte capacity is more than 'largeObjectThreshold'

832−-- then it is rounded up to the closest power of 2.

946+{-# DEPRECATED resize "Please use grow instead." #-}

947+{-# INLINE resize #-}

948+resize :: forall m a. (MonadIO m, Unbox a) =>

949+ Int -> MutArray a -> m (MutArray a)

950+resize = grow

951+

952+-- | Like 'grow' but if the requested byte capacity is more than

953+-- 'largeObjectThreshold' then it is rounded up to the closest power of 2.

833954--

834955-- /Pre-release/

835−{-# INLINE resizeExp #-}

836−resizeExp :: forall m a. (MonadIO m, Unbox a) =>

956+{-# INLINE growExp #-}

957+growExp :: forall m a. (MonadIO m, Unbox a) =>

837958 Int -> MutArray a -> m (MutArray a)

838−resizeExp nElems arr@MutArray{..} = do

959+growExp nElems arr@MutArray{..} = do

839960 let req = roundUpLargeArray (SIZE_OF(a) * nElems)

840961 req1 =

841962 if req > largeObjectThreshold

842963 then roundUpToPower2 req

843964 else req

844− len = arrEnd - arrStart

845− if req1 < len

965+ cap = arrBound - arrStart

966+ if req1 < cap

846967 then return arr

847968 else realloc req1 arr

848969

970+{-# DEPRECATED resizeExp "Please use growExp instead." #-}

971+{-# INLINE resizeExp #-}

972+resizeExp :: forall m a. (MonadIO m, Unbox a) =>

973+ Int -> MutArray a -> m (MutArray a)

974+resizeExp = growExp

975+

849976-- | Resize the allocated memory to drop any reserved free space at the end of

850977-- the array and reallocate it to reduce wastage.

851978--

@@ -890,7 +1017,7 @@ rightSize arr@MutArray{..} = do

8901017snocNewEnd :: (MonadIO m, Unbox a) => Int -> MutArray a -> a -> m (MutArray a)

8911018snocNewEnd newEnd arr@MutArray{..} x = liftIO $ do

8921019 assert (newEnd <= arrBound) (return ())

893− pokeByteIndex arrEnd arrContents x

1020+ pokeAt arrEnd arrContents x

8941021 return $ arr {arrEnd = newEnd}

8951022

8961023-- | Really really unsafe, appends the element into the first array, may

@@ -910,7 +1037,7 @@ snocUnsafe arr@MutArray{..} = snocNewEnd (INDEX_NEXT(arrEnd,a)) arr

9101037{-# INLINE snocMay #-}

9111038snocMay :: forall m a. (MonadIO m, Unbox a) =>

9121039 MutArray a -> a -> m (Maybe (MutArray a))

913−snocMay arr@MutArray{..} x = liftIO $ do

1040+snocMay arr@MutArray{..} x = do

9141041 let newEnd = INDEX_NEXT(arrEnd,a)

9151042 if newEnd <= arrBound

9161043 then Just <$> snocNewEnd newEnd arr x

@@ -924,7 +1051,7 @@ snocWithRealloc :: forall m a. (MonadIO m, Unbox a) =>

9241051 -> a

9251052 -> m (MutArray a)

9261053snocWithRealloc sizer arr x = do

927− arr1 <- liftIO $ reallocWith "snocWith" sizer (SIZE_OF(a)) arr

1054+ arr1 <- reallocWith "snocWith" sizer (SIZE_OF(a)) arr

9281055 snocUnsafe arr1 x

9291056

9301057-- | @snocWith sizer arr elem@ mutates @arr@ to append @elem@. The length of

@@ -946,7 +1073,7 @@ snocWith :: forall m a. (MonadIO m, Unbox a) =>

9461073 -> MutArray a

9471074 -> a

9481075 -> m (MutArray a)

949−snocWith allocSize arr x = liftIO $ do

1076+snocWith allocSize arr x = do

9501077 let newEnd = INDEX_NEXT(arrEnd arr,a)

9511078 if newEnd <= arrBound arr

9521079 then snocNewEnd newEnd arr x

@@ -990,6 +1117,126 @@ snoc = snocWith f

9901117 then oldSize * 2

9911118 else roundUpToPower2 oldSize * 2

9921119

1120+-------------------------------------------------------------------------------

1121+-- Serialization/Deserialization

1122+-------------------------------------------------------------------------------

1123+

1124+{-# INLINE pokeNewEnd #-}

1125+pokeNewEnd :: (MonadIO m, Unbox a) =>

1126+ Int -> MutArray Word8 -> a -> m (MutArray Word8)

1127+pokeNewEnd newEnd arr@MutArray{..} x = liftIO $ do

1128+ assert (newEnd <= arrBound) (return ())

1129+ liftIO $ pokeAt arrEnd arrContents x

1130+ return $ arr {arrEnd = newEnd}

1131+

1132+-- | Really really unsafe, unboxes a Haskell type and appends the resulting

1133+-- bytes to the byte array, may cause silent data corruption or if you are

1134+-- lucky a segfault if the array does not have enough space to append the

1135+-- element.

1136+--

1137+-- /Internal/

1138+{-# INLINE pokeAppendUnsafe #-}

1139+pokeAppendUnsafe :: forall m a. (MonadIO m, Unbox a) =>

1140+ MutArray Word8 -> a -> m (MutArray Word8)

1141+pokeAppendUnsafe arr@MutArray{..} = pokeNewEnd (arrEnd + SIZE_OF(a)) arr

1142+

1143+-- | Skip the specified number of bytes in the array. The data in the skipped

1144+-- region remains uninitialzed.

1145+{-# INLINE pokeSkipUnsafe #-}

1146+pokeSkipUnsafe :: Int -> MutArray Word8 -> MutArray Word8

1147+pokeSkipUnsafe n arr@MutArray{..} = do

1148+ let newEnd = arrEnd + n

1149+ in assert (newEnd <= arrBound) (arr {arrEnd = newEnd})

1150+

1151+-- | Like 'pokeAppend' but does not grow the array when pre-allocated array

1152+-- capacity becomes full.

1153+--

1154+-- /Internal/

1155+{-# INLINE pokeAppendMay #-}

1156+pokeAppendMay :: forall m a. (MonadIO m, Unbox a) =>

1157+ MutArray Word8 -> a -> m (Maybe (MutArray Word8))

1158+pokeAppendMay arr@MutArray{..} x = liftIO $ do

1159+ let newEnd = arrEnd + SIZE_OF(a)

1160+ if newEnd <= arrBound

1161+ then Just <$> pokeNewEnd newEnd arr x

1162+ else return Nothing

1163+

1164+{-# NOINLINE pokeWithRealloc #-}

1165+pokeWithRealloc :: forall m a. (MonadIO m, Unbox a) =>

1166+ (Int -> Int)

1167+ -> MutArray Word8

1168+ -> a

1169+ -> m (MutArray Word8)

1170+pokeWithRealloc sizer arr x = do

1171+ arr1 <- liftIO $ reallocWith "pokeWithRealloc" sizer (SIZE_OF(a)) arr

1172+ pokeAppendUnsafe arr1 x

1173+

1174+{-# INLINE pokeAppendWith #-}

1175+pokeAppendWith :: forall m a. (MonadIO m, Unbox a) =>

1176+ (Int -> Int)

1177+ -> MutArray Word8

1178+ -> a

1179+ -> m (MutArray Word8)

1180+pokeAppendWith allocSize arr x = liftIO $ do

1181+ let newEnd = arrEnd arr + SIZE_OF(a)

1182+ if newEnd <= arrBound arr

1183+ then pokeNewEnd newEnd arr x

1184+ else pokeWithRealloc allocSize arr x

1185+

1186+-- | Unbox a Haskell type and append the resulting bytes to a mutable byte

1187+-- array. The array is grown exponentially when more space is needed.

1188+--

1189+-- Definition:

1190+--

1191+-- >>> pokeAppend arr x = MutArray.castUnsafe <$> MutArray.snoc (MutArray.castUnsafe arr) x

1192+--

1193+{-# INLINE pokeAppend #-}

1194+pokeAppend :: forall m a. (MonadIO m, Unbox a) =>

1195+ MutArray Word8 -> a -> m (MutArray Word8)

1196+pokeAppend = pokeAppendWith f

1197+

1198+ where

1199+

1200+ f oldSize =

1201+ if isPower2 oldSize

1202+ then oldSize * 2

1203+ else roundUpToPower2 oldSize * 2

1204+

1205+-- | Really really unsafe, create a Haskell value from an unboxed byte array,

1206+-- does not check if the array is big enough, may return garbage or if you are

1207+-- lucky may cause a segfault.

1208+--

1209+-- /Internal/

1210+{-# INLINE peekUnconsUnsafe #-}

1211+peekUnconsUnsafe :: forall m a. (MonadIO m, Unbox a) =>

1212+ MutArray Word8 -> m (a, MutArray Word8)

1213+peekUnconsUnsafe MutArray{..} = do

1214+ let start1 = arrStart + SIZE_OF(a)

1215+ assert (start1 <= arrEnd) (return ())

1216+ liftIO $ do

1217+ r <- peekAt arrStart arrContents

1218+ return (r, MutArray arrContents start1 arrEnd arrBound)

1219+

1220+-- | Discard the specified number of bytes in the array.

1221+{-# INLINE peekSkipUnsafe #-}

1222+peekSkipUnsafe :: Int -> MutArray Word8 -> MutArray Word8

1223+peekSkipUnsafe n MutArray{..} =

1224+ let start1 = arrStart + n

1225+ in assert (start1 <= arrEnd) (MutArray arrContents start1 arrEnd arrBound)

1226+

1227+-- | Create a Haskell value from its unboxed representation from the head of a

1228+-- byte array, return the value and the remaining array.

1229+{-# INLINE peekUncons #-}

1230+peekUncons :: forall m a. (MonadIO m, Unbox a) =>

1231+ MutArray Word8 -> m (Maybe a, MutArray Word8)

1232+peekUncons arr@MutArray{..} = do

1233+ let start1 = arrStart + SIZE_OF(a)

1234+ if start1 > arrEnd

1235+ then return (Nothing, arr)

1236+ else liftIO $ do

1237+ r <- peekAt arrStart arrContents

1238+ return (Just r, MutArray arrContents start1 arrEnd arrBound)

1239+

9931240-------------------------------------------------------------------------------

9941241-- Random reads

9951242-------------------------------------------------------------------------------

@@ -1004,7 +1251,7 @@ getIndexUnsafe :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m a

10041251getIndexUnsafe i MutArray{..} = do

10051252 let index = INDEX_OF(arrStart,i,a)

10061253 assert (i >= 0 && INDEX_VALID(index,arrEnd,a)) (return ())

1007− liftIO $ peekByteIndex index arrContents

1254+ liftIO $ peekAt index arrContents

10081255

10091256-- | /O(1)/ Lookup the element at the given index. Index starts from 0.

10101257--

@@ -1013,7 +1260,7 @@ getIndex :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (Maybe a)

10131260getIndex i MutArray{..} = do

10141261 let index = INDEX_OF(arrStart,i,a)

10151262 if i >= 0 && INDEX_VALID(index,arrEnd,a)

1016− then liftIO $ Just <$> peekByteIndex index arrContents

1263+ then liftIO $ Just <$> peekAt index arrContents

10171264 else return Nothing

10181265

10191266-- | /O(1)/ Lookup the element at the given index from the end of the array.

@@ -1026,21 +1273,16 @@ getIndexRev :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m a

10261273getIndexRev i MutArray{..} = do

10271274 let index = RINDEX_OF(arrEnd,i,a)

10281275 if i >= 0 && index >= arrStart

1029− then liftIO $ peekByteIndex index arrContents

1276+ then liftIO $ peekAt index arrContents

10301277 else invalidIndex "getIndexRev" i

10311278

10321279data GetIndicesState contents start end st =

10331280 GetIndicesState contents start end st

10341281

1035−-- | Given an unfold that generates array indices, read the elements on those

1036−-- indices from the supplied MutArray. An error is thrown if an index is out of

1037−-- bounds.

1038−--

1039−-- /Pre-release/

1040−{-# INLINE getIndicesD #-}

1041−getIndicesD :: (Monad m, Unbox a) =>

1282+{-# INLINE indexReaderWith #-}

1283+indexReaderWith :: (Monad m, Unbox a) =>

10421284 (forall b. IO b -> m b) -> D.Stream m Int -> Unfold m (MutArray a) a

1043−getIndicesD liftio (D.Stream stepi sti) = Unfold step inject

1285+indexReaderWith liftio (D.Stream stepi sti) = Unfold step inject

10441286

10451287 where

10461288

@@ -1059,9 +1301,26 @@ getIndicesD liftio (D.Stream stepi sti) = Unfold step inject

10591301 D.Skip s -> return $ D.Skip (GetIndicesState contents start end s)

10601302 D.Stop -> return D.Stop

10611303

1304+{-# DEPRECATED getIndicesWith "Please use indexReaderWith instead." #-}

1305+{-# INLINE getIndicesWith #-}

1306+getIndicesWith :: (Monad m, Unbox a) =>

1307+ (forall b. IO b -> m b) -> D.Stream m Int -> Unfold m (MutArray a) a

1308+getIndicesWith = indexReaderWith

1309+

1310+-- | Given an unfold that generates array indices, read the elements on those

1311+-- indices from the supplied MutArray. An error is thrown if an index is out of

1312+-- bounds.

1313+--

1314+-- /Pre-release/

1315+{-# INLINE indexReader #-}

1316+indexReader :: (MonadIO m, Unbox a) => Stream m Int -> Unfold m (MutArray a) a

1317+indexReader = indexReaderWith liftIO

1318+

1319+-- XXX DO NOT REMOVE, change the signature to use Stream instead of unfold

1320+{-# DEPRECATED getIndices "Please use indexReader instead." #-}

10621321{-# INLINE getIndices #-}

10631322getIndices :: (MonadIO m, Unbox a) => Stream m Int -> Unfold m (MutArray a) a

1064−getIndices = getIndicesD liftIO

1323+getIndices = indexReader

10651324

10661325-------------------------------------------------------------------------------

10671326-- Subarrays

@@ -1166,7 +1425,7 @@ partitionBy f arr@MutArray{..} = liftIO $ do

11661425

11671426 -- Invariant low < high on entry, and on return as well

11681427 moveHigh low high = do

1169− h <- peekByteIndex high arrContents

1428+ h <- peekAt high arrContents

11701429 if f h

11711430 then

11721431 -- Correctly classified, continue the loop

@@ -1186,7 +1445,7 @@ partitionBy f arr@MutArray{..} = liftIO $ do

11861445 -- low <= high

11871446 -- Both low and high are valid locations within the array

11881447 go low high = do

1189− l <- peekByteIndex low arrContents

1448+ l <- peekAt low arrContents

11901449 if f l

11911450 then

11921451 -- low is wrongly classified

@@ -1197,8 +1456,8 @@ partitionBy f arr@MutArray{..} = liftIO $ do

11971456 case r of

11981457 Nothing -> return low

11991458 Just (high1, h) -> do -- low < high1

1200− pokeByteIndex low arrContents h

1201− pokeByteIndex high1 arrContents l

1459+ pokeAt low arrContents h

1460+ pokeAt high1 arrContents l

12021461 let low1 = INDEX_NEXT(low,a)

12031462 high2 = INDEX_PREV(high1,a)

12041463 if low1 <= high2

@@ -1332,7 +1591,7 @@ chunksOfAs ps n (D.Stream step state) =

13321591 r <- step (adaptState gst) st

13331592 case r of

13341593 D.Yield x s -> do

1335− liftIO $ pokeByteIndex end contents x

1594+ liftIO $ pokeAt end contents x

13361595 let end1 = INDEX_NEXT(end,a)

13371596 return $

13381597 if end1 >= bound

@@ -1351,10 +1610,12 @@ chunksOfAs ps n (D.Stream step state) =

13511610

13521611 step' _ GroupFinish = return D.Stop

13531612

1354−-- | @chunksOf n stream@ groups the input stream into a stream of

1355−-- arrays of size n.

1613+-- | @chunksOf n stream@ groups the elements in the input stream into arrays of

1614+-- @n@ elements each.

1615+--

1616+-- Same as the following but may be more efficient:

13561617--

1357−-- @chunksOf n = StreamD.foldMany (MutArray.writeN n)@

1618+-- >>> chunksOf n = Stream.foldMany (MutArray.createOf n)

13581619--

13591620-- /Pre-release/

13601621{-# INLINE_NORMAL chunksOf #-}

@@ -1363,17 +1624,29 @@ chunksOf :: forall m a. (MonadIO m, Unbox a)

13631624-- XXX the idiomatic implementation leads to large regression in the D.reverse'

13641625-- benchmark. It seems it has difficulty producing optimized code when

13651626-- converting to StreamK. Investigate GHC optimizations.

1366−-- chunksOf n = D.foldMany (writeN n)

1627+-- chunksOf n = D.foldMany (createOf n)

13671628chunksOf = chunksOfAs Unpinned

13681629

13691630-- | Like 'chunksOf' but creates pinned arrays.

13701631{-# INLINE_NORMAL pinnedChunksOf #-}

13711632pinnedChunksOf :: forall m a. (MonadIO m, Unbox a)

13721633 => Int -> D.Stream m a -> D.Stream m (MutArray a)

1373−-- pinnedChunksOf n = D.foldMany (pinnedWriteN n)

1634+-- pinnedChunksOf n = D.foldMany (pinnedCreateOf n)

13741635pinnedChunksOf = chunksOfAs Pinned

13751636

1376−-- XXX This should take a PinnedState

1637+-- | When we are buffering a stream of unknown size into an array we do not

1638+-- know how much space to pre-allocate. So we start with the min size and emit

1639+-- the array then keep on doubling the size every time. Thus we do not need to

1640+-- guess the optimum chunk size.

1641+--

1642+-- We can incorporate this in chunksOfAs if the additional size parameter does

1643+-- not impact perf.

1644+--

1645+{-# INLINE _chunksOfRange #-}

1646+_chunksOfRange :: -- (MonadIO m, Unbox a) =>

1647+ PinnedState -> Int -> Int -> D.Stream m a -> D.Stream m (MutArray a)

1648+_chunksOfRange _ps _low _hi = undefined

1649+

13771650-- XXX buffer to a list instead?

13781651-- | Buffer the stream into arrays in memory.

13791652{-# INLINE arrayStreamKFromStreamDAs #-}

@@ -1391,16 +1664,10 @@ data FlattenState s contents a =

13911664 OuterLoop s

13921665 | InnerLoop s contents !Int !Int

13931666

1394−-- | Use the "reader" unfold instead.

1395−--

1396−-- @flattenArrays = unfoldMany reader@

1397−--

1398−-- We can try this if there are any fusion issues in the unfold.

1399−--

1400−{-# INLINE_NORMAL flattenArrays #-}

1401−flattenArrays :: forall m a. (MonadIO m, Unbox a)

1402− => D.Stream m (MutArray a) -> D.Stream m a

1403−flattenArrays (D.Stream step state) = D.Stream step' (OuterLoop state)

1667+{-# INLINE_NORMAL concatWith #-}

1668+concatWith :: forall m a. (Monad m, Unbox a)

1669+ => (forall b. IO b -> m b) -> D.Stream m (MutArray a) -> D.Stream m a

1670+concatWith liftio (D.Stream step state) = D.Stream step' (OuterLoop state)

14041671

14051672 where

14061673

@@ -1417,19 +1684,30 @@ flattenArrays (D.Stream step state) = D.Stream step' (OuterLoop state)

14171684 return $ D.Skip $ OuterLoop st

14181685

14191686 step' _ (InnerLoop st contents p end) = do

1420− x <- liftIO $ peekByteIndex p contents

1687+ x <- liftio $ peekAt p contents

14211688 return $ D.Yield x (InnerLoop st contents (INDEX_NEXT(p,a)) end)

14221689

1423−-- | Use the "readerRev" unfold instead.

1690+-- | Use the "reader" unfold instead.

14241691--

1425−-- @flattenArrays = unfoldMany readerRev@

1692+-- @concat = unfoldMany reader@

14261693--

14271694-- We can try this if there are any fusion issues in the unfold.

14281695--

1429−{-# INLINE_NORMAL flattenArraysRev #-}

1430−flattenArraysRev :: forall m a. (MonadIO m, Unbox a)

1696+{-# INLINE_NORMAL concat #-}

1697+concat :: forall m a. (MonadIO m, Unbox a)

14311698 => D.Stream m (MutArray a) -> D.Stream m a

1432−flattenArraysRev (D.Stream step state) = D.Stream step' (OuterLoop state)

1699+concat = concatWith liftIO

1700+

1701+{-# DEPRECATED flattenArrays "Please use \"unfoldMany reader\" instead." #-}

1702+{-# INLINE flattenArrays #-}

1703+flattenArrays :: forall m a. (MonadIO m, Unbox a)

1704+ => D.Stream m (MutArray a) -> D.Stream m a

1705+flattenArrays = concat

1706+

1707+{-# INLINE_NORMAL concatRevWith #-}

1708+concatRevWith :: forall m a. (Monad m, Unbox a)

1709+ => (forall b. IO b -> m b) -> D.Stream m (MutArray a) -> D.Stream m a

1710+concatRevWith liftio (D.Stream step state) = D.Stream step' (OuterLoop state)

14331711

14341712 where

14351713

@@ -1447,16 +1725,33 @@ flattenArraysRev (D.Stream step state) = D.Stream step' (OuterLoop state)

14471725 return $ D.Skip $ OuterLoop st

14481726

14491727 step' _ (InnerLoop st contents p start) = do

1450− x <- liftIO $ peekByteIndex p contents

1728+ x <- liftio $ peekAt p contents

14511729 let cur = INDEX_PREV(p,a)

14521730 return $ D.Yield x (InnerLoop st contents cur start)

14531731

1732+-- | Use the "readerRev" unfold instead.

1733+--

1734+-- @concat = unfoldMany readerRev@

1735+--

1736+-- We can try this if there are any fusion issues in the unfold.

1737+--

1738+{-# INLINE_NORMAL concatRev #-}

1739+concatRev :: forall m a. (MonadIO m, Unbox a)

1740+ => D.Stream m (MutArray a) -> D.Stream m a

1741+concatRev = concatRevWith liftIO

1742+

1743+{-# DEPRECATED flattenArraysRev "Please use \"unfoldMany readerRev\" instead." #-}

1744+{-# INLINE flattenArraysRev #-}

1745+flattenArraysRev :: forall m a. (MonadIO m, Unbox a)

1746+ => D.Stream m (MutArray a) -> D.Stream m a

1747+flattenArraysRev = concatRev

1748+

14541749-------------------------------------------------------------------------------

14551750-- Unfolds

14561751-------------------------------------------------------------------------------

14571752

14581753data ArrayUnsafe a = ArrayUnsafe

1459− {-# UNPACK #-} !MutableByteArray -- contents

1754+ {-# UNPACK #-} !MutByteArray -- contents

14601755 {-# UNPACK #-} !Int -- index 1

14611756 {-# UNPACK #-} !Int -- index 2

14621757

@@ -1486,7 +1781,7 @@ producerWith liftio = Producer step (return . toArrayUnsafe) extract

14861781 -- few actions for correctness and execution order sanity. We want

14871782 -- the peek to occur right here and not lazily at some later point

14881783 -- because we want the peek to be ordered with respect to the touch.

1489− !x <- liftio $ peekByteIndex cur contents

1784+ !x <- liftio $ peekAt cur contents

14901785 return $ D.Yield x (ArrayUnsafe contents (INDEX_NEXT(cur,a)) end)

14911786

14921787 extract = return . fromArrayUnsafe

@@ -1517,7 +1812,7 @@ readerRevWith liftio = Unfold step inject

15171812 {-# INLINE_LATE step #-}

15181813 step (ArrayUnsafe _ start p) | p < start = return D.Stop

15191814 step (ArrayUnsafe contents start p) = do

1520− !x <- liftio $ peekByteIndex p contents

1815+ !x <- liftio $ peekAt p contents

15211816 return $ D.Yield x (ArrayUnsafe contents start (INDEX_PREV(p,a)))

15221817

15231818-- | Unfold an array into a stream in reverse order.

@@ -1548,7 +1843,7 @@ toListFB c n MutArray{..} = go arrStart

15481843 -- evaluated/written to before we peek at them.

15491844 -- XXX

15501845 let !x = unsafeInlineIO $ do

1551− r <- peekByteIndex arrContents p

1846+ r <- peekAt arrContents p

15521847 return r

15531848 in c x (go (PTR_NEXT(p,a)))

15541849-}

@@ -1565,21 +1860,21 @@ toList MutArray{..} = liftIO $ go arrStart

15651860

15661861 go p | assert (p <= arrEnd) (p == arrEnd) = return []

15671862 go p = do

1568− x <- peekByteIndex p arrContents

1863+ x <- peekAt p arrContents

15691864 (:) x <$> go (INDEX_NEXT(p,a))

15701865

1571−{-# INLINE_NORMAL toStreamDWith #-}

1572−toStreamDWith ::

1866+{-# INLINE_NORMAL toStreamWith #-}

1867+toStreamWith ::

15731868 forall m a. (Monad m, Unbox a)

15741869 => (forall b. IO b -> m b) -> MutArray a -> D.Stream m a

1575−toStreamDWith liftio MutArray{..} = D.Stream step arrStart

1870+toStreamWith liftio MutArray{..} = D.Stream step arrStart

15761871

15771872 where

15781873

15791874 {-# INLINE_LATE step #-}

15801875 step _ p | assert (p <= arrEnd) (p == arrEnd) = return D.Stop

15811876 step _ p = liftio $ do

1582− r <- peekByteIndex p arrContents

1877+ r <- peekAt p arrContents

15831878 return $ D.Yield r (INDEX_NEXT(p,a))

15841879

15851880-- | Convert a 'MutArray' into a stream.

@@ -1588,7 +1883,7 @@ toStreamDWith liftio MutArray{..} = D.Stream step arrStart

15881883--

15891884{-# INLINE_NORMAL read #-}

15901885read :: forall m a. (MonadIO m, Unbox a) => MutArray a -> D.Stream m a

1591−read = toStreamDWith liftIO

1886+read = toStreamWith liftIO

15921887

15931888{-# INLINE toStreamKWith #-}

15941889toStreamKWith ::

@@ -1600,18 +1895,18 @@ toStreamKWith liftio MutArray{..} = go arrStart

16001895

16011896 go p | assert (p <= arrEnd) (p == arrEnd) = K.nil

16021897 | otherwise =

1603− let elemM = peekByteIndex p arrContents

1898+ let elemM = peekAt p arrContents

16041899 in liftio elemM `K.consM` go (INDEX_NEXT(p,a))

16051900

16061901{-# INLINE toStreamK #-}

16071902toStreamK :: forall m a. (MonadIO m, Unbox a) => MutArray a -> StreamK m a

16081903toStreamK = toStreamKWith liftIO

16091904

1610−{-# INLINE_NORMAL toStreamDRevWith #-}

1611−toStreamDRevWith ::

1905+{-# INLINE_NORMAL toStreamRevWith #-}

1906+toStreamRevWith ::

16121907 forall m a. (Monad m, Unbox a)

16131908 => (forall b. IO b -> m b) -> MutArray a -> D.Stream m a

1614−toStreamDRevWith liftio MutArray{..} =

1909+toStreamRevWith liftio MutArray{..} =

16151910 let p = INDEX_PREV(arrEnd,a)

16161911 in D.Stream step p

16171912

@@ -1620,7 +1915,7 @@ toStreamDRevWith liftio MutArray{..} =

16201915 {-# INLINE_LATE step #-}

16211916 step _ p | p < arrStart = return D.Stop

16221917 step _ p = liftio $ do

1623− r <- peekByteIndex p arrContents

1918+ r <- peekAt p arrContents

16241919 return $ D.Yield r (INDEX_PREV(p,a))

16251920

16261921-- | Convert a 'MutArray' into a stream in reverse order.

@@ -1629,7 +1924,7 @@ toStreamDRevWith liftio MutArray{..} =

16291924--

16301925{-# INLINE_NORMAL readRev #-}

16311926readRev :: forall m a. (MonadIO m, Unbox a) => MutArray a -> D.Stream m a

1632−readRev = toStreamDRevWith liftIO

1927+readRev = toStreamRevWith liftIO

16331928

16341929{-# INLINE toStreamKRevWith #-}

16351930toStreamKRevWith ::

@@ -1643,7 +1938,7 @@ toStreamKRevWith liftio MutArray {..} =

16431938

16441939 go p | p < arrStart = K.nil

16451940 | otherwise =

1646− let elemM = peekByteIndex p arrContents

1941+ let elemM = peekAt p arrContents

16471942 in liftio elemM `K.consM` go (INDEX_PREV(p,a))

16481943

16491944{-# INLINE toStreamKRev #-}

@@ -1678,7 +1973,7 @@ foldr f z arr = D.foldr f z $ read arr

16781973

16791974-- XXX Keep the bound intact to not lose any free space? Perf impact?

16801975

1681−-- | @writeAppendNUnsafe n arr@ appends up to @n@ input items to the supplied

1976+-- | @unsafeAppendN n arr@ appends up to @n@ input items to the supplied

16821977-- array.

16831978--

16841979-- Unsafe: Do not drive the fold beyond @n@ elements, it will lead to memory

@@ -1687,13 +1982,12 @@ foldr f z arr = D.foldr f z $ read arr

16871982-- Any free space left in the array after appending @n@ elements is lost.

16881983--

16891984-- /Internal/

1690−{-# INLINE_NORMAL writeAppendNUnsafe #-}

1691−writeAppendNUnsafe :: forall m a. (MonadIO m, Unbox a) =>

1985+{-# INLINE_NORMAL unsafeAppendN #-}

1986+unsafeAppendN :: forall m a. (MonadIO m, Unbox a) =>

16921987 Int

16931988 -> m (MutArray a)

16941989 -> Fold m a (MutArray a)

1695−writeAppendNUnsafe n action =

1696− fmap fromArrayUnsafe $ FL.foldlM' step initial

1990+unsafeAppendN n action = fmap fromArrayUnsafe $ FL.foldlM' step initial

16971991

16981992 where

16991993

@@ -1706,38 +2000,59 @@ writeAppendNUnsafe n action =

17062000 -- otherwise we lose that space.

17072001 arr1 <-

17082002 if free < needed

1709− then noinline reallocWith "writeAppendNUnsafeWith" (+ needed) needed arr

2003+ then noinline reallocWith "unsafeAppendN" (+ needed) needed arr

17102004 else return arr

17112005 return $ toArrayUnsafe arr1

17122006

17132007 step (ArrayUnsafe contents start end) x = do

1714− liftIO $ pokeByteIndex end contents x

2008+ liftIO $ pokeAt end contents x

17152009 return $ ArrayUnsafe contents start (INDEX_NEXT(end,a))

17162010

2011+{-# DEPRECATED writeAppendNUnsafe "Please use unsafeAppendN instead." #-}

2012+{-# INLINE writeAppendNUnsafe #-}

2013+writeAppendNUnsafe :: forall m a. (MonadIO m, Unbox a) =>

2014+ Int

2015+ -> m (MutArray a)

2016+ -> Fold m a (MutArray a)

2017+writeAppendNUnsafe = unsafeAppendN

2018+

17172019-- | Append @n@ elements to an existing array. Any free space left in the array

17182020-- after appending @n@ elements is lost.

17192021--

1720−-- >>> writeAppendN n initial = Fold.take n (MutArray.writeAppendNUnsafe n initial)

2022+-- >>> appendN n initial = Fold.take n (MutArray.unsafeAppendN n initial)

17212023--

1722−{-# INLINE_NORMAL writeAppendN #-}

2024+{-# INLINE_NORMAL appendN #-}

2025+appendN :: forall m a. (MonadIO m, Unbox a) =>

2026+ Int -> m (MutArray a) -> Fold m a (MutArray a)

2027+appendN n initial = FL.take n (unsafeAppendN n initial)

2028+

2029+-- XXX Deprecate in major

2030+-- {-# DEPRECATED writeAppendN "Please use appendN instead." #-}

2031+{-# INLINE writeAppendN #-}

17232032writeAppendN :: forall m a. (MonadIO m, Unbox a) =>

17242033 Int -> m (MutArray a) -> Fold m a (MutArray a)

1725−writeAppendN n initial = FL.take n (writeAppendNUnsafe n initial)

2034+writeAppendN = appendN

17262035

1727−-- | @writeAppendWith realloc action@ mutates the array generated by @action@ to

2036+-- | @appendWith realloc action@ mutates the array generated by @action@ to

17282037-- append the input stream. If there is no reserved space available in the

17292038-- array it is reallocated to a size in bytes determined by @realloc oldSize@,

17302039-- where @oldSize@ is the current size of the array in bytes.

17312040--

17322041-- Note that the returned array may be a mutated version of original array.

17332042--

1734−-- >>> writeAppendWith sizer = Fold.foldlM' (MutArray.snocWith sizer)

2043+-- >>> appendWith sizer = Fold.foldlM' (MutArray.snocWith sizer)

17352044--

17362045-- /Pre-release/

2046+{-# INLINE appendWith #-}

2047+appendWith :: forall m a. (MonadIO m, Unbox a) =>

2048+ (Int -> Int) -> m (MutArray a) -> Fold m a (MutArray a)

2049+appendWith sizer = FL.foldlM' (snocWith sizer)

2050+

2051+{-# DEPRECATED writeAppendWith "Please use appendWith instead." #-}

17372052{-# INLINE writeAppendWith #-}

17382053writeAppendWith :: forall m a. (MonadIO m, Unbox a) =>

17392054 (Int -> Int) -> m (MutArray a) -> Fold m a (MutArray a)

1740−writeAppendWith sizer = FL.foldlM' (snocWith sizer)

2055+writeAppendWith = appendWith

17412056

17422057-- | @append action@ mutates the array generated by @action@ to append the

17432058-- input stream. If there is no reserved space available in the array it is

@@ -1745,70 +2060,101 @@ writeAppendWith sizer = FL.foldlM' (snocWith sizer)

17452060--

17462061-- Note that the returned array may be a mutated version of original array.

17472062--

1748−-- >>> writeAppend = MutArray.writeAppendWith (* 2)

2063+-- >>> append = MutArray.appendWith (* 2)

17492064--

2065+{-# INLINE append #-}

2066+append :: forall m a. (MonadIO m, Unbox a) =>

2067+ m (MutArray a) -> Fold m a (MutArray a)

2068+append = appendWith (* 2)

2069+

2070+-- XXX Deprecate in major

2071+-- {-# DEPRECATED writeAppend "Please use append instead." #-}

17502072{-# INLINE writeAppend #-}

17512073writeAppend :: forall m a. (MonadIO m, Unbox a) =>

17522074 m (MutArray a) -> Fold m a (MutArray a)

1753−writeAppend = writeAppendWith (* 2)

2075+writeAppend = append

17542076

17552077-- XXX We can carry bound as well in the state to make sure we do not lose the

17562078-- remaining capacity. Need to check perf impact.

17572079--

1758−-- | Like 'writeNUnsafe' but takes a new array allocator @alloc size@ function

1759−-- as argument.

2080+-- | Like 'unsafeCreateOf' but takes a new array allocator @alloc size@

2081+-- function as argument.

17602082--

1761−-- >>> writeNWithUnsafe alloc n = MutArray.writeAppendNUnsafe (alloc n) n

2083+-- >>> unsafeCreateOfWith alloc n = MutArray.unsafeAppendN (alloc n) n

17622084--

17632085-- /Pre-release/

1764−{-# INLINE_NORMAL writeNWithUnsafe #-}

1765−writeNWithUnsafe :: forall m a. (MonadIO m, Unbox a)

2086+{-# INLINE_NORMAL unsafeCreateOfWith #-}

2087+unsafeCreateOfWith :: forall m a. (MonadIO m, Unbox a)

17662088 => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)

1767−writeNWithUnsafe alloc n = fromArrayUnsafe <$> FL.foldlM' step initial

2089+unsafeCreateOfWith alloc n = fromArrayUnsafe <$> FL.foldlM' step initial

17682090

17692091 where

17702092

17712093 initial = toArrayUnsafe <$> alloc (max n 0)

17722094

17732095 step (ArrayUnsafe contents start end) x = do

1774− liftIO $ pokeByteIndex end contents x

2096+ liftIO $ pokeAt end contents x

17752097 return

17762098 $ ArrayUnsafe contents start (INDEX_NEXT(end,a))

17772099

2100+{-# DEPRECATED writeNWithUnsafe "Please use unsafeCreateOfWith instead." #-}

2101+{-# INLINE writeNWithUnsafe #-}

2102+writeNWithUnsafe :: forall m a. (MonadIO m, Unbox a)

2103+ => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)

2104+writeNWithUnsafe = unsafeCreateOfWith

2105+

17782106{-# INLINE_NORMAL writeNUnsafeAs #-}

17792107writeNUnsafeAs :: forall m a. (MonadIO m, Unbox a)

17802108 => PinnedState -> Int -> Fold m a (MutArray a)

1781−writeNUnsafeAs ps = writeNWithUnsafe (newAs ps)

2109+writeNUnsafeAs ps = unsafeCreateOfWith (newAs ps)

17822110

1783−-- | Like 'writeN' but does not check the array bounds when writing. The fold

2111+-- | Like 'createOf' but does not check the array bounds when writing. The fold

17842112-- driver must not call the step function more than 'n' times otherwise it will

17852113-- corrupt the memory and crash. This function exists mainly because any

17862114-- conditional in the step function blocks fusion causing 10x performance

17872115-- slowdown.

17882116--

1789−-- >>> writeNUnsafe = MutArray.writeNWithUnsafe MutArray.new

2117+-- >>> unsafeCreateOf = MutArray.unsafeCreateOfWith MutArray.emptyOf

17902118--

1791−{-# INLINE_NORMAL writeNUnsafe #-}

2119+{-# INLINE_NORMAL unsafeCreateOf #-}

2120+unsafeCreateOf :: forall m a. (MonadIO m, Unbox a)

2121+ => Int -> Fold m a (MutArray a)

2122+unsafeCreateOf = writeNUnsafeAs Unpinned

2123+

2124+{-# DEPRECATED writeNUnsafe "Please use unsafeCreateOf instead." #-}

2125+{-# INLINE writeNUnsafe #-}

17922126writeNUnsafe :: forall m a. (MonadIO m, Unbox a)

17932127 => Int -> Fold m a (MutArray a)

1794−writeNUnsafe = writeNUnsafeAs Unpinned

2128+writeNUnsafe = unsafeCreateOf

17952129

1796−-- | Like 'writeNUnsafe' but creates a pinned array.

1797−{-# INLINE_NORMAL pinnedWriteNUnsafe #-}

2130+-- | Like 'unsafeCreateOf' but creates a pinned array.

2131+{-# INLINE_NORMAL unsafePinnedCreateOf #-}

2132+unsafePinnedCreateOf :: forall m a. (MonadIO m, Unbox a)

2133+ => Int -> Fold m a (MutArray a)

2134+unsafePinnedCreateOf = writeNUnsafeAs Pinned

2135+

2136+{-# DEPRECATED pinnedWriteNUnsafe "Please use unsafePinnedCreateOf instead." #-}

2137+{-# INLINE pinnedWriteNUnsafe #-}

17982138pinnedWriteNUnsafe :: forall m a. (MonadIO m, Unbox a)

17992139 => Int -> Fold m a (MutArray a)

1800−pinnedWriteNUnsafe = writeNUnsafeAs Pinned

2140+pinnedWriteNUnsafe = unsafePinnedCreateOf

18012141

1802−-- | @writeNWith alloc n@ folds a maximum of @n@ elements into an array

2142+-- | @createOfWith alloc n@ folds a maximum of @n@ elements into an array

18032143-- allocated using the @alloc@ function.

18042144--

1805−-- >>> writeNWith alloc n = Fold.take n (MutArray.writeNWithUnsafe alloc n)

1806−-- >>> writeNWith alloc n = MutArray.writeAppendN (alloc n) n

2145+-- >>> createOfWith alloc n = Fold.take n (MutArray.unsafeCreateOfWith alloc n)

2146+-- >>> createOfWith alloc n = MutArray.appendN (alloc n) n

18072147--

1808−{-# INLINE_NORMAL writeNWith #-}

2148+{-# INLINE_NORMAL createOfWith #-}

2149+createOfWith :: forall m a. (MonadIO m, Unbox a)

2150+ => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)

2151+createOfWith alloc n = FL.take n (unsafeCreateOfWith alloc n)

2152+

2153+-- {-# DEPRECATED writeNWith "Please use createOfWith instead." #-}

2154+{-# INLINE writeNWith #-}

18092155writeNWith :: forall m a. (MonadIO m, Unbox a)

18102156 => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)

1811−writeNWith alloc n = FL.take n (writeNWithUnsafe alloc n)

2157+writeNWith = createOfWith

18122158

18132159{-# INLINE_NORMAL writeNAs #-}

18142160writeNAs ::

@@ -1816,28 +2162,42 @@ writeNAs ::

18162162 => PinnedState

18172163 -> Int

18182164 -> Fold m a (MutArray a)

1819−writeNAs ps = writeNWith (newAs ps)

2165+writeNAs ps = createOfWith (newAs ps)

18202166

1821−-- | @writeN n@ folds a maximum of @n@ elements from the input stream to an

2167+-- | @createOf n@ folds a maximum of @n@ elements from the input stream to an

18222168-- 'MutArray'.

18232169--

1824−-- >>> writeN = MutArray.writeNWith MutArray.new

1825−-- >>> writeN n = Fold.take n (MutArray.writeNUnsafe n)

1826−-- >>> writeN n = MutArray.writeAppendN n (MutArray.new n)

2170+-- >>> createOf = MutArray.createOfWith MutArray.new

2171+-- >>> createOf n = Fold.take n (MutArray.unsafeCreateOf n)

2172+-- >>> createOf n = MutArray.appendN n (MutArray.emptyOf n)

18272173--

1828−{-# INLINE_NORMAL writeN #-}

2174+{-# INLINE_NORMAL createOf #-}

2175+createOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

2176+createOf = writeNAs Unpinned

2177+

2178+-- XXX Deprecate in major

2179+-- {-# DEPRECATED writeN "Please use createOf instead." #-}

2180+{-# INLINE writeN #-}

18292181writeN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

1830−writeN = writeNAs Unpinned

2182+writeN = createOf

2183+

2184+-- | Like 'createOf' but creates a pinned array.

2185+{-# INLINE_NORMAL pinnedCreateOf #-}

2186+pinnedCreateOf ::

2187+ forall m a. (MonadIO m, Unbox a)

2188+ => Int

2189+ -> Fold m a (MutArray a)

2190+pinnedCreateOf = writeNAs Pinned

18312191

1832−-- | Like 'writeN' but creates a pinned array.

1833−{-# INLINE_NORMAL pinnedWriteN #-}

2192+{-# DEPRECATED pinnedWriteN "Please use pinnedCreateOf instead." #-}

2193+{-# INLINE pinnedWriteN #-}

18342194pinnedWriteN ::

18352195 forall m a. (MonadIO m, Unbox a)

18362196 => Int

18372197 -> Fold m a (MutArray a)

1838−pinnedWriteN = writeNAs Pinned

2198+pinnedWriteN = pinnedCreateOf

18392199

1840−-- | Like writeNWithUnsafe but writes the array in reverse order.

2200+-- | Like unsafeCreateOfWith but writes the array in reverse order.

18412201--

18422202-- /Internal/

18432203{-# INLINE_NORMAL writeRevNWithUnsafe #-}

@@ -1854,11 +2214,11 @@ writeRevNWithUnsafe alloc n = fromArrayUnsafe <$> FL.foldlM' step initial

18542214

18552215 step (ArrayUnsafe contents start end) x = do

18562216 let ptr = INDEX_PREV(start,a)

1857− liftIO $ pokeByteIndex ptr contents x

2217+ liftIO $ pokeAt ptr contents x

18582218 return

18592219 $ ArrayUnsafe contents ptr end

18602220

1861−-- | Like writeNWith but writes the array in reverse order.

2221+-- | Like createOfWith but writes the array in reverse order.

18622222--

18632223-- /Internal/

18642224{-# INLINE_NORMAL writeRevNWith #-}

@@ -1866,31 +2226,36 @@ writeRevNWith :: forall m a. (MonadIO m, Unbox a)

18662226 => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)

18672227writeRevNWith alloc n = FL.take n (writeRevNWithUnsafe alloc n)

18682228

1869−-- | Like writeN but writes the array in reverse order.

2229+-- | Like 'createOf' but writes the array in reverse order.

18702230--

18712231-- /Pre-release/

1872−{-# INLINE_NORMAL writeRevN #-}

2232+{-# INLINE_NORMAL revCreateOf #-}

2233+revCreateOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

2234+revCreateOf = writeRevNWith new

2235+

2236+{-# DEPRECATED writeRevN "Please use revCreateOf instead." #-}

2237+{-# INLINE writeRevN #-}

18732238writeRevN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)

1874−writeRevN = writeRevNWith new

2239+writeRevN = revCreateOf

18752240

18762241-- | @pinnedWriteNAligned align n@ folds a maximum of @n@ elements from the

18772242-- input stream to a 'MutArray' aligned to the given size.

18782243--

1879−-- >>> pinnedWriteNAligned align = MutArray.writeNWith (MutArray.pinnedNewAligned align)

1880−-- >>> pinnedWriteNAligned align n = MutArray.writeAppendN n (MutArray.pinnedNewAligned align n)

2244+-- >>> pinnedWriteNAligned align = MutArray.createOfWith (MutArray.pinnedNewAligned align)

2245+-- >>> pinnedWriteNAligned align n = MutArray.appendN n (MutArray.pinnedNewAligned align n)

18812246--

18822247-- /Pre-release/

18832248--

18842249{-# INLINE_NORMAL pinnedWriteNAligned #-}

18852250pinnedWriteNAligned :: forall m a. (MonadIO m, Unbox a)

18862251 => Int -> Int -> Fold m a (MutArray a)

1887−pinnedWriteNAligned align = writeNWith (pinnedNewAligned align)

2252+pinnedWriteNAligned align = createOfWith (pinnedNewAligned align)

18882253

18892254-- XXX Buffer to a list instead?

1890−--

2255+

18912256-- | Buffer a stream into a stream of arrays.

18922257--

1893−-- >>> writeChunks n = Fold.many (MutArray.writeN n) Fold.toStreamK

2258+-- >>> buildChunks n = Fold.many (MutArray.createOf n) Fold.toStreamK

18942259--

18952260-- Breaking an array into an array stream can be useful to consume a large

18962261-- array sequentially such that memory of the array is released incrementatlly.

@@ -1899,37 +2264,43 @@ pinnedWriteNAligned align = writeNWith (pinnedNewAligned align)

18992264--

19002265-- /Unimplemented/

19012266--

1902−{-# INLINE_NORMAL writeChunks #-}

2267+{-# INLINE_NORMAL buildChunks #-}

2268+buildChunks :: (MonadIO m, Unbox a) =>

2269+ Int -> Fold m a (StreamK n (MutArray a))

2270+buildChunks n = FL.many (createOf n) FL.toStreamK

2271+

2272+{-# DEPRECATED writeChunks "Please use buildChunks instead." #-}

2273+{-# INLINE writeChunks #-}

19032274writeChunks :: (MonadIO m, Unbox a) =>

19042275 Int -> Fold m a (StreamK n (MutArray a))

1905−writeChunks n = FL.many (writeN n) FL.toStreamK

2276+writeChunks = buildChunks

19062277

19072278{-# INLINE_NORMAL writeWithAs #-}

19082279writeWithAs :: forall m a. (MonadIO m, Unbox a)

19092280 => PinnedState -> Int -> Fold m a (MutArray a)

1910−-- writeWithAs ps n = FL.rmapM rightSize $ writeAppendWith (* 2) (newAs ps n)

2281+-- writeWithAs ps n = FL.rmapM rightSize $ appendWith (* 2) (newAs ps n)

19112282writeWithAs ps elemCount =

19122283 FL.rmapM extract $ FL.foldlM' step initial

19132284

19142285 where

19152286

19162287 initial = do

1917− when (elemCount < 0) $ error "writeWith: elemCount is negative"

1918− liftIO $ newAs ps elemCount

2288+ when (elemCount < 0) $ error "createWith: elemCount is negative"

2289+ newAs ps elemCount

19192290

19202291 step arr@(MutArray _ start end bound) x

19212292 | INDEX_NEXT(end,a) > bound = do

19222293 let oldSize = end - start

19232294 newSize = max (oldSize * 2) 1

1924− arr1 <- liftIO $ reallocExplicit (SIZE_OF(a)) newSize arr

2295+ arr1 <- liftIO $ reallocExplicitAs ps (SIZE_OF(a)) newSize arr

19252296 snocUnsafe arr1 x

19262297 step arr x = snocUnsafe arr x

19272298

19282299 extract = liftIO . rightSize

19292300

1930−-- XXX Compare writeWith with fromStreamD which uses an array of streams

1931−-- implementation. We can write this using writeChunks above if that is faster.

1932−-- If writeWith is faster then we should use that to implement

2301+-- XXX Compare createWith with fromStreamD which uses an array of streams

2302+-- implementation. We can write this using buildChunks above if that is faster.

2303+-- If createWith is faster then we should use that to implement

19332304-- fromStreamD.

19342305--

19352306-- XXX The realloc based implementation needs to make one extra copy if we use

@@ -1944,39 +2315,56 @@ writeWithAs ps elemCount =

19442315-- XXX check if GHC's memory allocator is efficient enough. We can try the C

19452316-- malloc to compare against.

19462317

1947−-- | @writeWith minCount@ folds the whole input to a single array. The array

2318+-- | @createWith minCount@ folds the whole input to a single array. The array

19482319-- starts at a size big enough to hold minCount elements, the size is doubled

19492320-- every time the array needs to be grown.

19502321--

19512322-- /Caution! Do not use this on infinite streams./

19522323--

1953−-- >>> f n = MutArray.writeAppendWith (* 2) (MutArray.new n)

1954−-- >>> writeWith n = Fold.rmapM MutArray.rightSize (f n)

1955−-- >>> writeWith n = Fold.rmapM MutArray.fromArrayStreamK (MutArray.writeChunks n)

2324+-- >>> f n = MutArray.appendWith (* 2) (MutArray.emptyOf n)

2325+-- >>> createWith n = Fold.rmapM MutArray.rightSize (f n)

2326+-- >>> createWith n = Fold.rmapM MutArray.fromChunksK (MutArray.buildChunks n)

19562327--

19572328-- /Pre-release/

1958−{-# INLINE_NORMAL writeWith #-}

2329+{-# INLINE_NORMAL createWith #-}

2330+createWith :: forall m a. (MonadIO m, Unbox a)

2331+ => Int -> Fold m a (MutArray a)

2332+-- createWith n = FL.rmapM rightSize $ appendWith (* 2) (emptyOf n)

2333+createWith = writeWithAs Unpinned

2334+

2335+{-# DEPRECATED writeWith "Please use createWith instead." #-}

2336+{-# INLINE writeWith #-}

19592337writeWith :: forall m a. (MonadIO m, Unbox a)

19602338 => Int -> Fold m a (MutArray a)

1961−-- writeWith n = FL.rmapM rightSize $ writeAppendWith (* 2) (new n)

1962−writeWith = writeWithAs Unpinned

2339+writeWith = createWith

19632340

19642341-- | Fold the whole input to a single array.

19652342--

1966−-- Same as 'writeWith' using an initial array size of 'arrayChunkBytes' bytes

2343+-- Same as 'createWith' using an initial array size of 'arrayChunkBytes' bytes

19672344-- rounded up to the element size.

19682345--

19692346-- /Caution! Do not use this on infinite streams./

19702347--

2348+{-# INLINE create #-}

2349+create :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)

2350+create = createWith (allocBytesToElemCount (undefined :: a) arrayChunkBytes)

2351+

2352+-- XXX Deprecate in major

2353+-- {-# DEPRECATED write "Please use create instead." #-}

19712354{-# INLINE write #-}

19722355write :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)

1973−write = writeWith (allocBytesToElemCount (undefined :: a) arrayChunkBytes)

2356+write = create

2357+

2358+-- | Like 'create' but creates a pinned array.

2359+{-# INLINE pinnedCreate #-}

2360+pinnedCreate :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)

2361+pinnedCreate =

2362+ writeWithAs Pinned (allocBytesToElemCount (undefined :: a) arrayChunkBytes)

19742363

1975−-- | Like 'write' but creates a pinned array.

2364+{-# DEPRECATED pinnedWrite "Please use pinnedCreate instead." #-}

19762365{-# INLINE pinnedWrite #-}

19772366pinnedWrite :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)

1978−pinnedWrite =

1979− writeWithAs Pinned (allocBytesToElemCount (undefined :: a) arrayChunkBytes)

2367+pinnedWrite = pinnedCreate

19802368

19812369-------------------------------------------------------------------------------

19822370-- construct from streams, known size

@@ -1986,25 +2374,31 @@ pinnedWrite =

19862374fromStreamDNAs :: forall m a. (MonadIO m, Unbox a)

19872375 => PinnedState -> Int -> D.Stream m a -> m (MutArray a)

19882376fromStreamDNAs ps limit str = do

1989− (arr :: MutArray a) <- liftIO $ newAs ps limit

2377+ (arr :: MutArray a) <- newAs ps limit

19902378 end <- D.foldlM' (fwrite (arrContents arr)) (return $ arrEnd arr) $ D.take limit str

19912379 return $ arr {arrEnd = end}

19922380

19932381 where

19942382

19952383 fwrite arrContents ptr x = do

1996− liftIO $ pokeByteIndex ptr arrContents x

2384+ liftIO $ pokeAt ptr arrContents x

19972385 return $ INDEX_NEXT(ptr,a)

19982386

1999−-- | Use the 'writeN' fold instead.

2387+-- | Use the 'createOf' fold instead.

20002388--

2001−-- >>> fromStreamDN n = Stream.fold (MutArray.writeN n)

2389+-- >>> fromStreamN n = Stream.fold (MutArray.createOf n)

20022390--

2003−{-# INLINE_NORMAL fromStreamDN #-}

2391+{-# INLINE_NORMAL fromStreamN #-}

2392+fromStreamN :: forall m a. (MonadIO m, Unbox a)

2393+ => Int -> D.Stream m a -> m (MutArray a)

2394+-- fromStreamDN n = D.fold (createOf n)

2395+fromStreamN = fromStreamDNAs Unpinned

2396+

2397+{-# DEPRECATED fromStreamDN "Please use fromStreamN instead." #-}

2398+{-# INLINE fromStreamDN #-}

20042399fromStreamDN :: forall m a. (MonadIO m, Unbox a)

20052400 => Int -> D.Stream m a -> m (MutArray a)

2006−-- fromStreamDN n = D.fold (writeN n)

2007−fromStreamDN = fromStreamDNAs Unpinned

2401+fromStreamDN = fromStreamN

20082402

20092403-- | Create a 'MutArray' from the first N elements of a list. The array is

20102404-- allocated to size N, if the list terminates before N elements then the

@@ -2024,16 +2418,70 @@ pinnedFromListN n xs = fromStreamDNAs Pinned n $ D.fromList xs

20242418-- /Pre-release/

20252419{-# INLINE fromListRevN #-}

20262420fromListRevN :: (MonadIO m, Unbox a) => Int -> [a] -> m (MutArray a)

2027−fromListRevN n xs = D.fold (writeRevN n) $ D.fromList xs

2421+fromListRevN n xs = D.fold (revCreateOf n) $ D.fromList xs

2422+

2423+-- | Convert a pure stream in Identity monad to a mutable array.

2424+{-# INLINABLE fromPureStreamN #-}

2425+fromPureStreamN :: (MonadIO m, Unbox a) =>

2426+ Int -> Stream Identity a -> m (MutArray a)

2427+fromPureStreamN n xs =

2428+ D.fold (createOf n) $ D.morphInner (return . runIdentity) xs

20282429

20292430-- | Convert a pure stream in Identity monad to a mutable array.

20302431{-# INLINABLE fromPureStream #-}

20312432fromPureStream :: (MonadIO m, Unbox a) => Stream Identity a -> m (MutArray a)

20322433fromPureStream xs =

2033− liftIO $ D.fold write $ D.morphInner (return . runIdentity) xs

2434+ D.fold create $ D.morphInner (return . runIdentity) xs

2435+

2436+{-# INLINABLE fromPtrN #-}

2437+fromPtrN :: MonadIO m => Int -> Ptr Word8 -> m (MutArray Word8)

2438+fromPtrN len addr = do

2439+ -- memcpy is better than stream copy when the size is known.

2440+ -- XXX We can implement a stream copy in a similar way by streaming Word64

2441+ -- first and then remaining Word8.

2442+ arr <- new len

2443+ _ <- unsafeAsPtr arr

2444+ (\ptr -> liftIO $ c_memcpy ptr addr (fromIntegral len))

2445+ return (arr {arrEnd = len})

2446+

2447+{-# INLINABLE fromByteStr# #-}

2448+fromByteStr# :: MonadIO m => Addr# -> m (MutArray Word8)

2449+fromByteStr# addr = do

2450+ -- It is better to count the size first and allocate exact space.

2451+ -- Also, memcpy is better than stream copy when the size is known.

2452+ -- C strlen compares 4 bytes at a time, so is better than the stream

2453+ -- version. https://github.com/bminor/glibc/blob/master/string/strlen.c

2454+ -- XXX We can possibly use a stream of Word64 to do the same.

2455+ -- fromByteStr# addr = fromPureStream (D.fromByteStr# addr)

2456+ len <- liftIO $ c_strlen (Ptr addr)

2457+ let lenInt = fromIntegral len

2458+ arr <- new lenInt

2459+ _ <- unsafeAsPtr arr (\ptr -> liftIO $ c_memcpy ptr (Ptr addr) len)

2460+ return (arr {arrEnd = lenInt})

20342461

20352462-------------------------------------------------------------------------------

2036−-- convert stream to a single array

2463+-- convert a stream of arrays to a single array by reallocating and copying

2464+-------------------------------------------------------------------------------

2465+

2466+-- XXX Both of these implementations of splicing seem to perform equally well.

2467+-- We need to perform benchmarks over a range of sizes though.

2468+

2469+-- | Also see 'fromChunksK'.

2470+{-# INLINE fromChunksRealloced #-}

2471+fromChunksRealloced :: forall m a. (MonadIO m, Unbox a)

2472+ => Stream m (MutArray a) -> m (MutArray a)

2473+fromChunksRealloced s = do

2474+ res <- D.uncons s

2475+ case res of

2476+ Just (a, strm) -> do

2477+ arr <- D.foldlM' spliceExp (pure a) strm

2478+ -- Reallocation is exponential so there may be 50% empty space in

2479+ -- worst case. One more reallocation to reclaim the space.

2480+ rightSize arr

2481+ Nothing -> pure nil

2482+

2483+-------------------------------------------------------------------------------

2484+-- convert a stream of arrays to a single array by buffering arrays first

20372485-------------------------------------------------------------------------------

20382486

20392487{-# INLINE arrayStreamKLength #-}

@@ -2043,34 +2491,72 @@ arrayStreamKLength as = K.foldl' (+) 0 (K.map length as)

20432491-- | Convert an array stream to an array. Note that this requires peak memory

20442492-- that is double the size of the array stream.

20452493--

2494+{-# INLINE fromChunkskAs #-}

2495+fromChunkskAs :: (Unbox a, MonadIO m) =>

2496+ PinnedState -> StreamK m (MutArray a) -> m (MutArray a)

2497+fromChunkskAs ps as = do

2498+ len <- arrayStreamKLength as

2499+ arr <- newAs ps len

2500+ -- XXX is StreamK fold faster or StreamD fold?

2501+ K.foldlM' spliceUnsafe (pure arr) as

2502+ -- fromStreamDN len $ D.unfoldMany reader $ D.fromStreamK as

2503+

2504+-- XXX Need to compare this with fromChunks and fromChunkList and keep the

2505+-- fastest or simplest one if all are equally fast.

2506+

2507+-- | Convert an array stream to an array. Note that this requires peak memory

2508+-- that is double the size of the array stream.

2509+--

2510+-- Also see 'fromChunksRealloced'.

2511+--

2512+{-# INLINE fromChunksK #-}

2513+fromChunksK :: (Unbox a, MonadIO m) =>

2514+ StreamK m (MutArray a) -> m (MutArray a)

2515+fromChunksK = fromChunkskAs Unpinned

2516+

2517+{-# DEPRECATED fromArrayStreamK "Please use fromChunksK instead." #-}

20462518{-# INLINE fromArrayStreamK #-}

20472519fromArrayStreamK :: (Unbox a, MonadIO m) =>

20482520 StreamK m (MutArray a) -> m (MutArray a)

2049−fromArrayStreamK as = do

2050− len <- arrayStreamKLength as

2051− fromStreamDN len $ D.unfoldMany reader $ D.fromStreamK as

2521+fromArrayStreamK = fromChunksK

20522522

20532523{-# INLINE fromStreamDAs #-}

20542524fromStreamDAs ::

20552525 (MonadIO m, Unbox a) => PinnedState -> D.Stream m a -> m (MutArray a)

2056−fromStreamDAs ps m = arrayStreamKFromStreamDAs ps m >>= fromArrayStreamK

2526+fromStreamDAs ps m =

2527+ arrayStreamKFromStreamDAs Unpinned m >>= fromChunkskAs ps

20572528

2529+-- | Create an 'Array' from a stream. This is useful when we want to create a

2530+-- single array from a stream of unknown size. 'createOf' is at least twice

2531+-- as efficient when the size is already known.

2532+--

2533+-- Note that if the input stream is too large memory allocation for the array

2534+-- may fail. When the stream size is not known, `chunksOf` followed by

2535+-- processing of indvidual arrays in the resulting stream should be preferred.

2536+--

2537+-- /Pre-release/

2538+{-# INLINE fromStream #-}

2539+fromStream :: (MonadIO m, Unbox a) => Stream m a -> m (MutArray a)

2540+fromStream = fromStreamDAs Unpinned

2541+

2542+-- fromStream (Stream m) = P.fold create m

20582543-- CAUTION: a very large number (millions) of arrays can degrade performance

20592544-- due to GC overhead because we need to buffer the arrays before we flatten

20602545-- all the arrays.

20612546--

2062−-- XXX Compare if this is faster or "fold write".

2547+-- XXX Compare if this is faster or "fold create".

20632548--

20642549-- | We could take the approach of doubling the memory allocation on each

20652550-- overflow. This would result in more or less the same amount of copying as in

20662551-- the chunking approach. However, if we have to shrink in the end then it may

20672552-- result in an extra copy of the entire data.

20682553--

2069−-- >>> fromStreamD = StreamD.fold MutArray.write

2554+-- >>> fromStreamD = StreamD.fold MutArray.create

20702555--

20712556{-# INLINE fromStreamD #-}

2557+{-# DEPRECATED fromStreamD "Please use fromStream instead." #-}

20722558fromStreamD :: (MonadIO m, Unbox a) => D.Stream m a -> m (MutArray a)

2073−fromStreamD = fromStreamDAs Unpinned

2559+fromStreamD = fromStream

20742560

20752561-- | Create a 'MutArray' from a list. The list must be of finite size.

20762562--

@@ -2095,22 +2581,44 @@ fromListRev xs = fromListRevN (Prelude.length xs) xs

20952581-- Cloning

20962582-------------------------------------------------------------------------------

20972583

2584+-- Arrays are aligned on 64-bit boundaries. The fastest way to copy an array is

2585+-- to unsafeCast it to Word64, read it, write it to Word64 array and unsafeCast

2586+-- it again. We can use SIMD read/write as well.

2587+

20982588{-# INLINE cloneAs #-}

2099−cloneAs :: MonadIO m => PinnedState -> MutArray a -> m (MutArray a)

2589+cloneAs ::

2590+ ( MonadIO m

2591+#ifdef DEVBUILD

2592+ , Unbox a

2593+#endif

2594+ )

2595+ => PinnedState -> MutArray a -> m (MutArray a)

21002596cloneAs ps src =

2101− liftIO $ do

2597+ do

21022598 let startSrc = arrStart src

21032599 srcLen = arrEnd src - startSrc

2104− newArrContents <- Unboxed.newBytesAs ps srcLen

2105− putSliceUnsafe (arrContents src) startSrc newArrContents 0 srcLen

2600+ newArrContents <-

2601+ Unboxed.cloneSliceUnsafeAs ps startSrc srcLen (arrContents src)

21062602 return $ MutArray newArrContents 0 srcLen srcLen

21072603

21082604{-# INLINE clone #-}

2109−clone :: MonadIO m => MutArray a -> m (MutArray a)

2605+clone ::

2606+ ( MonadIO m

2607+#ifdef DEVBUILD

2608+ , Unbox a

2609+#endif

2610+ )

2611+ => MutArray a -> m (MutArray a)

21102612clone = cloneAs Unpinned

21112613

21122614{-# INLINE pinnedClone #-}

2113−pinnedClone :: MonadIO m => MutArray a -> m (MutArray a)

2615+pinnedClone ::

2616+ ( MonadIO m

2617+#ifdef DEVBUILD

2618+ , Unbox a

2619+#endif

2620+ )

2621+ => MutArray a -> m (MutArray a)

21142622pinnedClone = cloneAs Pinned

21152623

21162624-------------------------------------------------------------------------------

@@ -2125,17 +2633,16 @@ spliceCopy :: forall m a. MonadIO m =>

21252633 Unbox a =>

21262634#endif

21272635 MutArray a -> MutArray a -> m (MutArray a)

2128−spliceCopy arr1 arr2 = liftIO $ do

2636+spliceCopy arr1 arr2 = do

21292637 let start1 = arrStart arr1

21302638 start2 = arrStart arr2

21312639 len1 = arrEnd arr1 - start1

21322640 len2 = arrEnd arr2 - start2

2133− let newLen = len1 + len2

2641+ let len = len1 + len2

21342642 newArrContents <-

21352643 if Unboxed.isPinned (arrContents arr1)

2136− then Unboxed.pinnedNewBytes newLen

2137− else Unboxed.newBytes newLen

2138− let len = len1 + len2

2644+ then liftIO $ Unboxed.pinnedNew len

2645+ else liftIO $ Unboxed.new len

21392646 putSliceUnsafe (arrContents arr1) start1 newArrContents 0 len1

21402647 putSliceUnsafe (arrContents arr2) start2 newArrContents len1 len2

21412648 return $ MutArray newArrContents 0 len len

@@ -2147,7 +2654,7 @@ spliceCopy arr1 arr2 = liftIO $ do

21472654spliceUnsafe :: MonadIO m =>

21482655 MutArray a -> MutArray a -> m (MutArray a)

21492656spliceUnsafe dst src =

2150− liftIO $ do

2657+ do

21512658 let startSrc = arrStart src

21522659 srcLen = arrEnd src - startSrc

21532660 endDst = arrEnd dst

@@ -2169,7 +2676,7 @@ spliceWith :: forall m a. (MonadIO m, Unbox a) =>

21692676 (Int -> Int -> Int) -> MutArray a -> MutArray a -> m (MutArray a)

21702677spliceWith sizer dst@(MutArray _ start end bound) src = do

21712678{-

2172− let f = writeAppendWith (`sizer` byteLength src) (return dst)

2679+ let f = appendWith (`sizer` byteLength src) (return dst)

21732680 in D.fold f (toStreamD src)

21742681-}

21752682 assert (end <= bound) (return ())

@@ -2185,7 +2692,7 @@ spliceWith sizer dst@(MutArray _ start end bound) src = do

21852692 $ "splice: newSize is less than the total size "

21862693 ++ "of arrays being appended. Please check the "

21872694 ++ "sizer function passed."

2188− liftIO $ realloc newSizeInBytes dst

2695+ realloc newSizeInBytes dst

21892696 else return dst

21902697 spliceUnsafe dst1 src

21912698

@@ -2222,14 +2729,24 @@ spliceExp = spliceWith (\l1 l2 -> max (l1 * 2) (l1 + l2))

22222729-- Splitting

22232730-------------------------------------------------------------------------------

22242731

2732+-- | Generate a stream of array slices using a predicate. The array element

2733+-- matching the predicate is dropped.

2734+--

2735+-- /Pre-release/

2736+{-# INLINE splitOn #-}

2737+splitOn :: (MonadIO m, Unbox a) =>

2738+ (a -> Bool) -> MutArray a -> Stream m (MutArray a)

2739+splitOn predicate arr =

2740+ fmap (\(i, len) -> getSliceUnsafe i len arr)

2741+ $ D.indexOnSuffix predicate (read arr)

2742+

22252743-- | Drops the separator byte

22262744{-# INLINE breakOn #-}

22272745breakOn :: MonadIO m

22282746 => Word8 -> MutArray Word8 -> m (MutArray Word8, Maybe (MutArray Word8))

2229−breakOn sep arr@MutArray{..} = asPtrUnsafe arr $ \p -> liftIO $ do

2230− -- XXX Instead of using asPtrUnsafe (pinning memory) we can pass unlifted

2231− -- Addr# to memchr and it should be safe (from ghc 8.4).

2747+breakOn sep arr@MutArray{..} = unsafeAsPtr arr $ \p -> liftIO $ do

22322748 -- XXX We do not need memchr here, we can use a Haskell equivalent.

2749+ -- Need efficient stream based primitives that work on Word64.

22332750 loc <- c_memchr p sep (fromIntegral $ byteLength arr)

22342751 let sepIndex = loc `minusPtr` p

22352752 return $

@@ -2250,32 +2767,41 @@ breakOn sep arr@MutArray{..} = asPtrUnsafe arr $ \p -> liftIO $ do

22502767 }

22512768 )

22522769

2770+-- | Like 'splitAt' but does not check whether the index is valid.

2771+--

2772+{-# INLINE unsafeSplitAt #-}

2773+unsafeSplitAt :: forall a. Unbox a =>

2774+ Int -> MutArray a -> (MutArray a, MutArray a)

2775+unsafeSplitAt i MutArray{..} =

2776+ let off = i * SIZE_OF(a)

2777+ p = arrStart + off

2778+ in ( MutArray

2779+ { arrContents = arrContents

2780+ , arrStart = arrStart

2781+ , arrEnd = p

2782+ , arrBound = p

2783+ }

2784+ , MutArray

2785+ { arrContents = arrContents

2786+ , arrStart = p

2787+ , arrEnd = arrEnd

2788+ , arrBound = arrBound

2789+ }

2790+ )

2791+

22532792-- | Create two slices of an array without copying the original array. The

22542793-- specified index @i@ is the first index of the second slice.

22552794--

2795+{-# INLINE splitAt #-}

22562796splitAt :: forall a. Unbox a => Int -> MutArray a -> (MutArray a, MutArray a)

2257−splitAt i arr@MutArray{..} =

2797+splitAt i arr =

22582798 let maxIndex = length arr - 1

22592799 in if i < 0

22602800 then error "sliceAt: negative array index"

22612801 else if i > maxIndex

22622802 then error $ "sliceAt: specified array index " ++ show i

22632803 ++ " is beyond the maximum index " ++ show maxIndex

2264− else let off = i * SIZE_OF(a)

2265− p = arrStart + off

2266− in ( MutArray

2267− { arrContents = arrContents

2268− , arrStart = arrStart

2269− , arrEnd = p

2270− , arrBound = p

2271− }

2272− , MutArray

2273− { arrContents = arrContents

2274− , arrStart = p

2275− , arrEnd = arrEnd

2276− , arrBound = arrBound

2277− }

2278− )

2804+ else unsafeSplitAt i arr

22792805

22802806-------------------------------------------------------------------------------

22812807-- Casting

@@ -2325,13 +2851,16 @@ cast arr =

23252851-- guarantees that unsafe calls will be performed in the calling thread. Making

23262852-- it safe to pass heap-allocated objects to unsafe functions.

23272853

2328−-- Unsafe because of direct pointer operations. The user must ensure that they

2329−-- are writing within the legal bounds of the array. Should we just name it

2330−-- asPtr, the unsafety is implicit for any pointer operations. And we are safe

2331−-- from Haskell perspective because we will be pinning the memory.

2854+-- XXX Should we just name it asPtr, the unsafety is implicit for any pointer

2855+-- operations. And we are safe from Haskell perspective because we will be

2856+-- pinning the memory.

2857+--

2858+-- XXX we cannot pass the length of the ptr here as in some cases it may not be

2859+-- available e.g. a null terminated C string. However, we can create another

2860+-- flavor of the API e.g. asPtrN.

23322861

2333−-- | Use an @MutArray a@ as @Ptr a@. This is useful when we want to pass an array

2334−-- as a pointer to some operating system call or to a "safe" FFI call.

2862+-- | Use a @MutArray a@ as @Ptr a@. This is useful when we want to pass an

2863+-- array as a pointer to some operating system call or to a "safe" FFI call.

23352864--

23362865-- If the array is not pinned it is copied to pinned memory before passing it

23372866-- to the monadic action.

@@ -2341,57 +2870,382 @@ cast arr =

23412870-- opportunistically before this operation occurs, to avoid the cost of a copy

23422871-- if possible.

23432872--

2344−-- /Unsafe/

2873+-- /Unsafe/ because of direct pointer operations. The user must ensure that

2874+-- they are writing within the legal bounds of the array.

23452875--

23462876-- /Pre-release/

23472877--

2878+{-# INLINE unsafePinnedAsPtr #-}

2879+unsafePinnedAsPtr :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b

2880+unsafePinnedAsPtr arr f =

2881+ Unboxed.unsafePinnedAsPtr

2882+ (arrContents arr) (\ptr -> f (ptr `plusPtr` arrStart arr))

2883+

2884+{-# DEPRECATED asPtrUnsafe "Please use unsafePinnedAsPtr instead." #-}

23482885{-# INLINE asPtrUnsafe #-}

23492886asPtrUnsafe :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b

2350−asPtrUnsafe arr f = do

2351− contents <- liftIO $ Unboxed.pin $ arrContents arr

2352− let !ptr = Ptr (byteArrayContents#

2353− (unsafeCoerce# (getMutableByteArray# contents)))

2887+asPtrUnsafe = unsafePinnedAsPtr

23542888

2355− r <- f (ptr `plusPtr` arrStart arr)

2356− liftIO $ touch contents

2357− return r

2889+{-# INLINE unsafeAsPtr #-}

2890+unsafeAsPtr :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b

2891+unsafeAsPtr arr f =

2892+ Unboxed.unsafeAsPtr

2893+ (arrContents arr) (\ptr -> f (ptr `plusPtr` arrStart arr))

23582894

23592895-------------------------------------------------------------------------------

23602896-- Equality

23612897-------------------------------------------------------------------------------

23622898

2363−-- | Compare the length of the arrays. If the length is equal, compare the

2364−-- lexicographical ordering of two underlying byte arrays otherwise return the

2365−-- result of length comparison.

2899+-- | Byte compare two arrays. Compare the length of the arrays. If the length

2900+-- is equal, compare the lexicographical ordering of two underlying byte arrays

2901+-- otherwise return the result of length comparison.

2902+--

2903+-- /Unsafe/: Note that the 'Unbox' instance of sum types with constructors of

2904+-- different sizes may leave some memory uninitialized which can make byte

2905+-- comparison unreliable.

23662906--

23672907-- /Pre-release/

2908+{-# INLINE byteCmp #-}

2909+byteCmp :: MonadIO m => MutArray a -> MutArray a -> m Ordering

2910+byteCmp arr1 arr2 = do

2911+ let marr1 = getMutableByteArray# (arrContents arr1)

2912+ marr2 = getMutableByteArray# (arrContents arr2)

2913+ !(I# st1#) = arrStart arr1

2914+ !(I# st2#) = arrStart arr2

2915+ !(I# len#) = byteLength arr1

2916+ case compare (byteLength arr1) (byteLength arr2) of

2917+ EQ -> do

2918+ r <- liftIO $ IO $ \s# ->

2919+ let res =

2920+ I#

2921+ (compareByteArrays#

2922+ (unsafeCoerce# marr1)

2923+ st1#

2924+ (unsafeCoerce# marr2)

2925+ st2#

2926+ len#)

2927+ in (# s#, res #)

2928+ return $ compare r 0

2929+ x -> return x

2930+

23682931{-# INLINE cmp #-}

2932+{-# DEPRECATED cmp "Please use byteCmp instead." #-}

23692933cmp :: MonadIO m => MutArray a -> MutArray a -> m Ordering

2370−cmp arr1 arr2 =

2371− liftIO

2372− $ do

2373− let marr1 = getMutableByteArray# (arrContents arr1)

2374− marr2 = getMutableByteArray# (arrContents arr2)

2375− !(I# st1#) = arrStart arr1

2376− !(I# st2#) = arrStart arr2

2377− !(I# len#) = byteLength arr1

2378− case compare (byteLength arr1) (byteLength arr2) of

2379− EQ -> do

2380− r <- liftIO $ IO $ \s# ->

2381− let res =

2382− I#

2383− (compareByteArrays#

2384− (unsafeCoerce# marr1)

2385− st1#

2386− (unsafeCoerce# marr2)

2387− st2#

2388− len#)

2389− in (# s#, res #)

2390− return $ compare r 0

2391− x -> return x

2934+cmp = byteCmp

2935+

2936+-- | Byte equality of two arrays.

2937+--

2938+-- >>> byteEq arr1 arr2 = (==) EQ $ MArray.byteCmp arr1 arr2

2939+--

2940+-- /Unsafe/: See 'byteCmp'.

2941+{-# INLINE byteEq #-}

2942+byteEq :: MonadIO m => MutArray a -> MutArray a -> m Bool

2943+byteEq arr1 arr2 = fmap (EQ ==) $ byteCmp arr1 arr2

2944+

2945+-------------------------------------------------------------------------------

2946+-- Compact

2947+-------------------------------------------------------------------------------

2948+

2949+-- Note: LE versions avoid an extra copy compared to GE. LE parser trades

2950+-- backtracking one array in lieu of avoiding a copy. However, LE and GE both

2951+-- can leave some memory unused. They can split the last array to fit it

2952+-- exactly in the space.

2953+

2954+{-# INLINE_NORMAL pCompactLeAs #-}

2955+pCompactLeAs ::

2956+ forall m a. (MonadIO m, Unbox a)

2957+ => PinnedState -> Int -> Parser (MutArray a) m (MutArray a)

2958+pCompactLeAs ps maxElems = Parser step initial extract

2959+

2960+ where

2961+

2962+ maxBytes = maxElems * SIZE_OF(a)

2963+

2964+ functionName = "Streamly.Internal.Data.MutArray.pCompactLE"

2965+

2966+ initial =

2967+ return

2968+ $ if maxElems <= 0

2969+ then error

2970+ $ functionName

2971+ ++ ": the size of arrays ["

2972+ ++ show maxElems ++ "] must be a natural number"

2973+ else Parser.IPartial Nothing

2974+

2975+ step Nothing arr =

2976+ return

2977+ $ let len = byteLength arr

2978+ in if len >= maxBytes

2979+ then Parser.Done 0 arr

2980+ else Parser.Partial 0 (Just arr)

2981+ -- XXX Split the last array to use the space more compactly.

2982+ step (Just buf) arr =

2983+ let len = byteLength buf + byteLength arr

2984+ in if len > maxBytes

2985+ then return $ Parser.Done 1 buf

2986+ else do

2987+ buf1 <-

2988+ if byteCapacity buf < maxBytes

2989+ then liftIO $ reallocExplicitAs

2990+ ps (SIZE_OF(a)) maxBytes buf

2991+ else return buf

2992+ buf2 <- spliceUnsafe buf1 arr

2993+ return $ Parser.Partial 0 (Just buf2)

2994+

2995+ extract Nothing = return $ Parser.Done 0 nil

2996+ extract (Just buf) = return $ Parser.Done 0 buf

2997+

2998+-- | Parser @pCompactLE maxElems@ coalesces adjacent arrays in the input stream

2999+-- only if the combined size would be less than or equal to @maxElems@

3000+-- elements. Note that it won't split an array if the original array is already

3001+-- larger than maxElems.

3002+--

3003+-- @maxElems@ must be greater than 0.

3004+--

3005+-- Generates unpinned arrays irrespective of the pinning status of input

3006+-- arrays.

3007+--

3008+-- /Internal/

3009+{-# INLINE pCompactLE #-}

3010+pCompactLE ::

3011+ forall m a. (MonadIO m, Unbox a)

3012+ => Int -> Parser (MutArray a) m (MutArray a)

3013+pCompactLE = pCompactLeAs Unpinned

3014+

3015+-- | Pinned version of 'pCompactLE'.

3016+{-# INLINE pPinnedCompactLE #-}

3017+pPinnedCompactLE ::

3018+ forall m a. (MonadIO m, Unbox a)

3019+ => Int -> Parser (MutArray a) m (MutArray a)

3020+pPinnedCompactLE = pCompactLeAs Pinned

3021+

3022+data SpliceState s arr

3023+ = SpliceInitial s

3024+ | SpliceBuffering s arr

3025+ | SpliceYielding arr (SpliceState s arr)

3026+ | SpliceFinish

3027+

3028+-- This mutates the first array (if it has space) to append values from the

3029+-- second one. This would work for immutable arrays as well because an

3030+-- immutable array never has additional space so a new array is allocated

3031+-- instead of mutating it.

3032+

3033+{-# INLINE_NORMAL compactLeAs #-}

3034+compactLeAs :: forall m a. (MonadIO m, Unbox a)

3035+ => PinnedState -> Int -> D.Stream m (MutArray a) -> D.Stream m (MutArray a)

3036+compactLeAs ps maxElems (D.Stream step state) =

3037+ D.Stream step' (SpliceInitial state)

3038+

3039+ where

3040+

3041+ maxBytes = maxElems * SIZE_OF(a)

3042+

3043+ functionName = "Streamly.Internal.Data.MutArray.rCompactLE"

3044+

3045+ {-# INLINE_LATE step' #-}

3046+ step' gst (SpliceInitial st) = do

3047+ when (maxElems <= 0) $

3048+ -- XXX we can pass the module string from the higher level API

3049+ error $ functionName ++ ": the size of arrays [" ++ show maxElems

3050+ ++ "] must be a natural number"

3051+ r <- step gst st

3052+ case r of

3053+ D.Yield arr s -> return $

3054+ let len = byteLength arr

3055+ in if len >= maxBytes

3056+ then D.Skip (SpliceYielding arr (SpliceInitial s))

3057+ else D.Skip (SpliceBuffering s arr)

3058+ D.Skip s -> return $ D.Skip (SpliceInitial s)

3059+ D.Stop -> return D.Stop

3060+

3061+ -- XXX Split the last array to use the space more compactly.

3062+ step' gst (SpliceBuffering st buf) = do

3063+ r <- step gst st

3064+ case r of

3065+ D.Yield arr s -> do

3066+ let len = byteLength buf + byteLength arr

3067+ if len > maxBytes

3068+ then return $

3069+ D.Skip (SpliceYielding buf (SpliceBuffering s arr))

3070+ else do

3071+ buf1 <- if byteCapacity buf < maxBytes

3072+ then liftIO $ reallocExplicitAs

3073+ ps (SIZE_OF(a)) maxBytes buf

3074+ else return buf

3075+ buf2 <- spliceUnsafe buf1 arr

3076+ return $ D.Skip (SpliceBuffering s buf2)

3077+ D.Skip s -> return $ D.Skip (SpliceBuffering s buf)

3078+ D.Stop -> return $ D.Skip (SpliceYielding buf SpliceFinish)

3079+

3080+ step' _ SpliceFinish = return D.Stop

3081+

3082+ step' _ (SpliceYielding arr next) = return $ D.Yield arr next

3083+

3084+

3085+{-# INLINE_NORMAL fCompactGeAs #-}

3086+fCompactGeAs ::

3087+ forall m a. (MonadIO m, Unbox a)

3088+ => PinnedState -> Int -> FL.Fold m (MutArray a) (MutArray a)

3089+fCompactGeAs ps minElems = Fold step initial extract extract

3090+

3091+ where

3092+

3093+ minBytes = minElems * SIZE_OF(a)

3094+

3095+ functionName = "Streamly.Internal.Data.MutArray.fCompactGE"

3096+

3097+ initial =

3098+ return

3099+ $ if minElems < 0

3100+ then error

3101+ $ functionName

3102+ ++ ": the size of arrays ["

3103+ ++ show minElems ++ "] must be a natural number"

3104+ else FL.Partial Nothing

3105+

3106+ step Nothing arr =

3107+ return

3108+ $ let len = byteLength arr

3109+ in if len >= minBytes

3110+ then FL.Done arr

3111+ else FL.Partial (Just arr)

3112+ -- XXX Buffer arrays as a list to avoid copy and reallocations

3113+ step (Just buf) arr = do

3114+ let len = byteLength buf + byteLength arr

3115+ buf1 <-

3116+ if byteCapacity buf < len

3117+ then liftIO $ reallocExplicitAs

3118+ ps (SIZE_OF(a)) (max minBytes len) buf

3119+ else return buf

3120+ buf2 <- spliceUnsafe buf1 arr

3121+ if len >= minBytes

3122+ then return $ FL.Done buf2

3123+ else return $ FL.Partial (Just buf2)

3124+

3125+ extract Nothing = return nil

3126+ extract (Just buf) = return buf

3127+

3128+-- | Fold @fCompactGE minElems@ coalesces adjacent arrays in the input stream

3129+-- until the size becomes greater than or equal to @minElems@.

3130+--

3131+-- Generates unpinned arrays irrespective of the pinning status of input

3132+-- arrays.

3133+{-# INLINE fCompactGE #-}

3134+fCompactGE ::

3135+ forall m a. (MonadIO m, Unbox a)

3136+ => Int -> FL.Fold m (MutArray a) (MutArray a)

3137+fCompactGE = fCompactGeAs Unpinned

3138+

3139+-- | Pinned version of 'fCompactGE'.

3140+{-# INLINE fPinnedCompactGE #-}

3141+fPinnedCompactGE ::

3142+ forall m a. (MonadIO m, Unbox a)

3143+ => Int -> FL.Fold m (MutArray a) (MutArray a)

3144+fPinnedCompactGE = fCompactGeAs Pinned

3145+

3146+{-# INLINE_NORMAL lCompactGeAs #-}

3147+lCompactGeAs :: forall m a. (MonadIO m, Unbox a)

3148+ => PinnedState -> Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()

3149+-- The fold version turns out to be a little bit slower.

3150+-- lCompactGeAs ps n = FL.many (fCompactGeAs ps n)

3151+lCompactGeAs ps minElems (Fold step1 initial1 _ final1) =

3152+ Fold step initial extract final

3153+

3154+ where

3155+

3156+ minBytes = minElems * SIZE_OF(a)

3157+

3158+ functionName = "Streamly.Internal.Data.MutArray.lCompactGE"

3159+

3160+ initial = do

3161+ when (minElems <= 0) $

3162+ -- XXX we can pass the module string from the higher level API

3163+ error $ functionName ++ ": the size of arrays ["

3164+ ++ show minElems ++ "] must be a natural number"

3165+

3166+ r <- initial1

3167+ return $ first (Tuple' Nothing) r

3168+

3169+ {-# INLINE runInner #-}

3170+ runInner len acc buf =

3171+ if len >= minBytes

3172+ then do

3173+ r <- step1 acc buf

3174+ case r of

3175+ FL.Done _ -> return $ FL.Done ()

3176+ FL.Partial s -> do

3177+ _ <- final1 s

3178+ res <- initial1

3179+ return $ first (Tuple' Nothing) res

3180+ else return $ FL.Partial $ Tuple' (Just buf) acc

3181+

3182+ step (Tuple' Nothing r1) arr =

3183+ runInner (byteLength arr) r1 arr

3184+

3185+ -- XXX Buffer arrays as a list to avoid copy and reallocations

3186+ step (Tuple' (Just buf) r1) arr = do

3187+ let len = byteLength buf + byteLength arr

3188+ buf1 <- if byteCapacity buf < len

3189+ then liftIO $ reallocExplicitAs

3190+ ps (SIZE_OF(a)) (max minBytes len) buf

3191+ else return buf

3192+ buf2 <- spliceUnsafe buf1 arr

3193+ runInner len r1 buf2

3194+

3195+ -- XXX Several folds do extract >=> final, therefore, we need to make final

3196+ -- return "m b" rather than using extract post it if we want extract to be

3197+ -- partial.

3198+ --

3199+ -- extract forces the pending buffer to be sent to the fold which is not

3200+ -- what we want.

3201+ extract _ = error "lCompactGE: not designed for scanning"

3202+

3203+ final (Tuple' Nothing r1) = final1 r1

3204+ final (Tuple' (Just buf) r1) = do

3205+ r <- step1 r1 buf

3206+ case r of

3207+ FL.Partial rr -> final1 rr

3208+ FL.Done _ -> return ()

3209+

3210+-- | Like 'compactGE' but for transforming folds instead of stream.

3211+--

3212+-- >>> lCompactGE n = Fold.many (MutArray.fCompactGE n)

3213+--

3214+-- Generates unpinned arrays irrespective of the pinning status of input

3215+-- arrays.

3216+{-# INLINE lCompactGE #-}

3217+lCompactGE :: forall m a. (MonadIO m, Unbox a)

3218+ => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()

3219+lCompactGE = lCompactGeAs Unpinned

3220+

3221+-- | Pinned version of 'lCompactGE'.

3222+{-# INLINE lPinnedCompactGE #-}

3223+lPinnedCompactGE :: forall m a. (MonadIO m, Unbox a)

3224+ => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()

3225+lPinnedCompactGE = lCompactGeAs Pinned

3226+

3227+-- | @compactGE n stream@ coalesces adjacent arrays in the @stream@ until

3228+-- the size becomes greater than or equal to @n@.

3229+--

3230+-- >>> compactGE n = Stream.foldMany (MutArray.fCompactGE n)

3231+--

3232+{-# INLINE compactGE #-}

3233+compactGE ::

3234+ (MonadIO m, Unbox a)

3235+ => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

3236+compactGE n = D.foldMany (fCompactGE n)

3237+

3238+-- | 'compactEQ n' coalesces adajacent arrays in the input stream to

3239+-- arrays of exact size @n@.

3240+--

3241+-- /Unimplemented/

3242+{-# INLINE compactEQ #-}

3243+compactEQ :: -- (MonadIO m, Unbox a) =>

3244+ Int -> Stream m (MutArray a) -> Stream m (MutArray a)

3245+compactEQ _n = undefined -- D.parseManyD (pCompactEQ n)

23923246

23933247-------------------------------------------------------------------------------

2394−-- NFData

3248+-- In-place mutation algorithms

23953249-------------------------------------------------------------------------------

23963250

23973251-- | Strip elements which match with predicate from both ends.

@@ -2420,7 +3274,7 @@ strip eq arr@MutArray{..} = liftIO $ do

24203274 getStart cur = do

24213275 if cur < arrEnd

24223276 then do

2423− r <- peekByteIndex cur arrContents

3277+ r <- peekAt cur arrContents

24243278 if eq r

24253279 then getStart (INDEX_NEXT(cur,a))

24263280 else return cur

@@ -2430,7 +3284,7 @@ strip eq arr@MutArray{..} = liftIO $ do

24303284 if cur > low

24313285 then do

24323286 let prev = INDEX_PREV(cur,a)

2433− r <- peekByteIndex prev arrContents

3287+ r <- peekAt prev arrContents

24343288 if eq r

24353289 then getLast prev low

24363290 else return cur

core/src/Streamly/Internal/Data/Serialize.hs → core/src/Streamly/Internal/Data/MutByteArray.hsrenamed+83−70

@@ -4,15 +4,24 @@

44{-# OPTIONS_GHC -fno-warn-orphans #-}

66-- |

7−-- Module : Streamly.Internal.Data.Serialize

7+-- Module : Streamly.Internal.Data.MutByteArray

99-- License : BSD3-3-Clause

1010-- Maintainer : streamly@composewell.com

1111-- Portability : GHC

1212--

1313

14−module Streamly.Internal.Data.Serialize

15− ( module Streamly.Internal.Data.Serialize.Type

14+module Streamly.Internal.Data.MutByteArray

15+ (

16+ -- * MutByteArray

17+ module Streamly.Internal.Data.MutByteArray.Type

18+ -- * Unbox

19+ , module Streamly.Internal.Data.Unbox

20+ , module Streamly.Internal.Data.Unbox.TH

21+ -- * Serialize

22+ , module Streamly.Internal.Data.Serialize.Type

23+ -- * Serialize TH

24+ , module Streamly.Internal.Data.Serialize.TH

1625 ) where

1726

1827--------------------------------------------------------------------------------

@@ -21,7 +30,6 @@ module Streamly.Internal.Data.Serialize

2130

2231import Data.Proxy (Proxy(..))

2332import Streamly.Internal.Data.Array (Array(..))

24−import Streamly.Internal.Data.Unbox (MutableByteArray(..))

2533import GHC.Exts (Int(..), sizeofByteArray#, unsafeCoerce#)

2634import GHC.Word (Word8)

2735

@@ -31,7 +39,11 @@ import GHC.Num.Integer (Integer(..))

3139import GHC.Integer.GMP.Internals (Integer(..), BigNat(..))

3240#endif

3341

42+import Streamly.Internal.Data.MutByteArray.Type

43+import Streamly.Internal.Data.Serialize.TH

3444import Streamly.Internal.Data.Serialize.Type

45+import Streamly.Internal.Data.Unbox

46+import Streamly.Internal.Data.Unbox.TH

3547

3648--------------------------------------------------------------------------------

3749-- Common instances

@@ -76,73 +88,73 @@ import Streamly.Internal.Data.Serialize.Type

7688-- $(Serialize.deriveSerialize ''Maybe)

7789instance Serialize a => Serialize (Maybe a) where

7890

79− {-# INLINE size #-}

80− size acc x =

91+ {-# INLINE addSizeTo #-}

92+ addSizeTo acc x =

8193 case x of

8294 Nothing -> (acc + 1)

83− Just field0 -> (size (acc + 1)) field0

95+ Just field0 -> (addSizeTo (acc + 1)) field0

8496

85− {-# INLINE deserialize #-}

86− deserialize initialOffset arr endOffset = do

87− (i0, tag) <- ((deserialize initialOffset) arr) endOffset

97+ {-# INLINE deserializeAt #-}

98+ deserializeAt initialOffset arr endOffset = do

99+ (i0, tag) <- ((deserializeAt initialOffset) arr) endOffset

88100 case tag :: Word8 of

89101 0 -> pure (i0, Nothing)

90− 1 -> do (i1, a0) <- ((deserialize i0) arr) endOffset

102+ 1 -> do (i1, a0) <- ((deserializeAt i0) arr) endOffset

91103 pure (i1, Just a0)

92104 _ -> error "Found invalid tag while peeking (Maybe a)"

93105

94− {-# INLINE serialize #-}

95− serialize initialOffset arr val =

106+ {-# INLINE serializeAt #-}

107+ serializeAt initialOffset arr val =

96108 case val of

97109 Nothing -> do

98− i0 <- ((serialize initialOffset) arr) (0 :: Word8)

110+ i0 <- ((serializeAt initialOffset) arr) (0 :: Word8)

99111 pure i0

100112 Just field0 -> do

101− i0 <- ((serialize initialOffset) arr) (1 :: Word8)

102− i1 <- ((serialize i0) arr) field0

113+ i0 <- ((serializeAt initialOffset) arr) (1 :: Word8)

114+ i1 <- ((serializeAt i0) arr) field0

103115 pure i1

104116

105117-- $(Serialize.deriveSerialize ''Either)

106118instance (Serialize a, Serialize b) => Serialize (Either a b) where

107119

108− {-# INLINE size #-}

109− size acc x =

120+ {-# INLINE addSizeTo #-}

121+ addSizeTo acc x =

110122 case x of

111− Left field0 -> (size (acc + 1)) field0

112− Right field0 -> (size (acc + 1)) field0

123+ Left field0 -> (addSizeTo (acc + 1)) field0

124+ Right field0 -> (addSizeTo (acc + 1)) field0

113125

114− {-# INLINE deserialize #-}

115− deserialize initialOffset arr endOffset = do

116− (i0, tag) <- ((deserialize initialOffset) arr) endOffset

126+ {-# INLINE deserializeAt #-}

127+ deserializeAt initialOffset arr endOffset = do

128+ (i0, tag) <- ((deserializeAt initialOffset) arr) endOffset

117129 case tag :: Word8 of

118− 0 -> do (i1, a0) <- ((deserialize i0) arr) endOffset

130+ 0 -> do (i1, a0) <- ((deserializeAt i0) arr) endOffset

119131 pure (i1, Left a0)

120− 1 -> do (i1, a0) <- ((deserialize i0) arr) endOffset

132+ 1 -> do (i1, a0) <- ((deserializeAt i0) arr) endOffset

121133 pure (i1, Right a0)

122134 _ -> error "Found invalid tag while peeking (Either a b)"

123135

124− {-# INLINE serialize #-}

125− serialize initialOffset arr val =

136+ {-# INLINE serializeAt #-}

137+ serializeAt initialOffset arr val =

126138 case val of

127139 Left field0 -> do

128− i0 <- ((serialize initialOffset) arr) (0 :: Word8)

129− i1 <- ((serialize i0) arr) field0

140+ i0 <- ((serializeAt initialOffset) arr) (0 :: Word8)

141+ i1 <- ((serializeAt i0) arr) field0

130142 pure i1

131143 Right field0 -> do

132− i0 <- ((serialize initialOffset) arr) (1 :: Word8)

133− i1 <- ((serialize i0) arr) field0

144+ i0 <- ((serializeAt initialOffset) arr) (1 :: Word8)

145+ i1 <- ((serializeAt i0) arr) field0

134146 pure i1

135147

136148instance Serialize (Proxy a) where

137149

138− {-# INLINE size #-}

139− size acc _ = (acc + 1)

150+ {-# INLINE addSizeTo #-}

151+ addSizeTo acc _ = (acc + 1)

140152

141− {-# INLINE deserialize #-}

142− deserialize initialOffset _ _ = pure ((initialOffset + 1), Proxy)

153+ {-# INLINE deserializeAt #-}

154+ deserializeAt initialOffset _ _ = pure ((initialOffset + 1), Proxy)

143155

144− {-# INLINE serialize #-}

145− serialize initialOffset _ _ = pure (initialOffset + 1)

156+ {-# INLINE serializeAt #-}

157+ serializeAt initialOffset _ _ = pure (initialOffset + 1)

146158

147159--------------------------------------------------------------------------------

148160-- Integer

@@ -156,39 +168,39 @@ data LiftedInteger

156168-- $(Serialize.deriveSerialize ''LiftedInteger)

157169instance Serialize LiftedInteger where

158170

159− {-# INLINE size #-}

160− size acc x =

171+ {-# INLINE addSizeTo #-}

172+ addSizeTo acc x =

161173 case x of

162− LIS field0 -> (size (acc + 1)) field0

163− LIP field0 -> (size (acc + 1)) field0

164− LIN field0 -> (size (acc + 1)) field0

174+ LIS field0 -> (addSizeTo (acc + 1)) field0

175+ LIP field0 -> (addSizeTo (acc + 1)) field0

176+ LIN field0 -> (addSizeTo (acc + 1)) field0

165177

166− {-# INLINE deserialize #-}

167− deserialize initialOffset arr endOffset = do

168− (i0, tag) <- ((deserialize initialOffset) arr) endOffset

178+ {-# INLINE deserializeAt #-}

179+ deserializeAt initialOffset arr endOffset = do

180+ (i0, tag) <- ((deserializeAt initialOffset) arr) endOffset

169181 case tag :: Word8 of

170− 0 -> do (i1, a0) <- ((deserialize i0) arr) endOffset

182+ 0 -> do (i1, a0) <- ((deserializeAt i0) arr) endOffset

171183 pure (i1, LIS a0)

172− 1 -> do (i1, a0) <- ((deserialize i0) arr) endOffset

184+ 1 -> do (i1, a0) <- ((deserializeAt i0) arr) endOffset

173185 pure (i1, LIP a0)

174− 2 -> do (i1, a0) <- ((deserialize i0) arr) endOffset

186+ 2 -> do (i1, a0) <- ((deserializeAt i0) arr) endOffset

175187 pure (i1, LIN a0)

176188 _ -> error "Found invalid tag while peeking (LiftedInteger)"

177189

178− {-# INLINE serialize #-}

179− serialize initialOffset arr val =

190+ {-# INLINE serializeAt #-}

191+ serializeAt initialOffset arr val =

180192 case val of

181193 LIS field0 -> do

182− i0 <- ((serialize initialOffset) arr) (0 :: Word8)

183− i1 <- ((serialize i0) arr) field0

194+ i0 <- ((serializeAt initialOffset) arr) (0 :: Word8)

195+ i1 <- ((serializeAt i0) arr) field0

184196 pure i1

185197 LIP field0 -> do

186− i0 <- ((serialize initialOffset) arr) (1 :: Word8)

187− i1 <- ((serialize i0) arr) field0

198+ i0 <- ((serializeAt initialOffset) arr) (1 :: Word8)

199+ i1 <- ((serializeAt i0) arr) field0

188200 pure i1

189201 LIN field0 -> do

190− i0 <- ((serialize initialOffset) arr) (2 :: Word8)

191− i1 <- ((serialize i0) arr) field0

202+ i0 <- ((serializeAt initialOffset) arr) (2 :: Word8)

203+ i1 <- ((serializeAt i0) arr) field0

192204 pure i1

193205

194206#if __GLASGOW_HASKELL__ >= 900

@@ -197,15 +209,15 @@ instance Serialize LiftedInteger where

197209liftInteger :: Integer -> LiftedInteger

198210liftInteger (IS x) = LIS (I# x)

199211liftInteger (IP x) =

200− LIP (Array (MutableByteArray (unsafeCoerce# x)) 0 (I# (sizeofByteArray# x)))

212+ LIP (Array (MutByteArray (unsafeCoerce# x)) 0 (I# (sizeofByteArray# x)))

201213liftInteger (IN x) =

202− LIN (Array (MutableByteArray (unsafeCoerce# x)) 0 (I# (sizeofByteArray# x)))

214+ LIN (Array (MutByteArray (unsafeCoerce# x)) 0 (I# (sizeofByteArray# x)))

203215

204216{-# INLINE unliftInteger #-}

205217unliftInteger :: LiftedInteger -> Integer

206218unliftInteger (LIS (I# x)) = IS x

207−unliftInteger (LIP (Array (MutableByteArray x) _ _)) = IP (unsafeCoerce# x)

208−unliftInteger (LIN (Array (MutableByteArray x) _ _)) = IN (unsafeCoerce# x)

219+unliftInteger (LIP (Array (MutByteArray x) _ _)) = IP (unsafeCoerce# x)

220+unliftInteger (LIN (Array (MutByteArray x) _ _)) = IN (unsafeCoerce# x)

209221

210222#else

211223

@@ -213,26 +225,27 @@ unliftInteger (LIN (Array (MutableByteArray x) _ _)) = IN (unsafeCoerce# x)

213225liftInteger :: Integer -> LiftedInteger

214226liftInteger (S# x) = LIS (I# x)

215227liftInteger (Jp# (BN# x)) =

216− LIP (Array (MutableByteArray (unsafeCoerce# x)) 0 (I# (sizeofByteArray# x)))

228+ LIP (Array (MutByteArray (unsafeCoerce# x)) 0 (I# (sizeofByteArray# x)))

217229liftInteger (Jn# (BN# x)) =

218− LIN (Array (MutableByteArray (unsafeCoerce# x)) 0 (I# (sizeofByteArray# x)))

230+ LIN (Array (MutByteArray (unsafeCoerce# x)) 0 (I# (sizeofByteArray# x)))

219231

220232{-# INLINE unliftInteger #-}

221233unliftInteger :: LiftedInteger -> Integer

222234unliftInteger (LIS (I# x)) = S# x

223−unliftInteger (LIP (Array (MutableByteArray x) _ _)) =

235+unliftInteger (LIP (Array (MutByteArray x) _ _)) =

224236 Jp# (BN# (unsafeCoerce# x))

225−unliftInteger (LIN (Array (MutableByteArray x) _ _)) =

237+unliftInteger (LIN (Array (MutByteArray x) _ _)) =

226238 Jn# (BN# (unsafeCoerce# x))

227239

228240#endif

229241

230242instance Serialize Integer where

231− {-# INLINE size #-}

232− size i a = size i (liftInteger a)

243+ {-# INLINE addSizeTo #-}

244+ addSizeTo i a = addSizeTo i (liftInteger a)

233245

234− {-# INLINE deserialize #-}

235− deserialize off arr end = fmap unliftInteger <$> deserialize off arr end

246+ {-# INLINE deserializeAt #-}

247+ deserializeAt off arr end =

248+ fmap unliftInteger <$> deserializeAt off arr end

236249

237− {-# INLINE serialize #-}

238− serialize off arr val = serialize off arr (liftInteger val)

250+ {-# INLINE serializeAt #-}

251+ serializeAt off arr val = serializeAt off arr (liftInteger val)

core/src/Streamly/Internal/Data/MutByteArray/Type.hsnew file+321

@@ -0,0 +1,321 @@

1+{-# LANGUAGE UnboxedTuples #-}

3+-- |

4+-- Module : Streamly.Internal.Data.MutByteArray.Type

6+-- License : BSD3-3-Clause

7+-- Maintainer : streamly@composewell.com

8+-- Stability : experimental

9+-- Portability : GHC

10+--

11+module Streamly.Internal.Data.MutByteArray.Type

12+ (

13+ -- ** MutByteArray

14+ MutByteArray(..)

15+ , MutableByteArray

16+ , getMutableByteArray# -- XXX getMutByteArray#

17+

18+ -- ** Pinning

19+ , PinnedState(..)

20+ , isPinned

21+ , pin

22+ , unpin

23+

24+ -- ** Allocation

25+ , empty

26+ , newBytesAs -- XXX should be removed

27+ , new

28+ , pinnedNew

29+ , pinnedNewAlignedBytes -- XXX should be removed

30+

31+ -- ** Access

32+ , sizeOfMutableByteArray -- XXX length

33+ , putSliceUnsafe

34+ , cloneSliceUnsafeAs

35+ , cloneSliceUnsafe

36+ , pinnedCloneSliceUnsafe

37+ , unsafePinnedAsPtr

38+ , unsafeAsPtr

39+

40+ -- ** Deprecated

41+ , asPtrUnsafe

42+ , nil

43+ ) where

44+

45+import Control.Monad.IO.Class (MonadIO(..))

46+#ifdef DEBUG

47+import Control.Monad (when)

48+import Debug.Trace (trace)

49+#endif

50+import GHC.Base (IO(..))

51+import System.IO.Unsafe (unsafePerformIO)

52+

53+import GHC.Exts

54+

55+--------------------------------------------------------------------------------

56+-- The ArrayContents type

57+--------------------------------------------------------------------------------

58+

59+data PinnedState

60+ = Pinned

61+ | Unpinned deriving (Show, Eq)

62+

63+-- XXX can use UnliftedNewtypes

64+

65+-- | A lifted mutable byte array type wrapping @MutableByteArray# RealWorld@.

66+-- This is a low level array used to back high level unboxed arrays and

67+-- serialized data.

68+data MutByteArray = MutByteArray (MutableByteArray# RealWorld)

69+

70+{-# DEPRECATED MutableByteArray "Please use MutByteArray instead" #-}

71+type MutableByteArray = MutByteArray

72+

73+{-# INLINE getMutableByteArray# #-}

74+getMutableByteArray# :: MutByteArray -> MutableByteArray# RealWorld

75+getMutableByteArray# (MutByteArray mbarr) = mbarr

76+

77+-- | Return the size of the array in bytes.

78+{-# INLINE sizeOfMutableByteArray #-}

79+sizeOfMutableByteArray :: MutByteArray -> IO Int

80+sizeOfMutableByteArray (MutByteArray arr) =

81+ IO $ \s ->

82+ case getSizeofMutableByteArray# arr s of

83+ (# s1, i #) -> (# s1, I# i #)

84+

85+{-# INLINE touch #-}

86+touch :: MutByteArray -> IO ()

87+touch (MutByteArray contents) =

88+ IO $ \s -> case touch# contents s of s' -> (# s', () #)

89+

90+-- XXX Some functions in this module are "IO" and others are "m", we need to

91+-- make it consistent.

92+

93+-- XXX We can provide another API for "unsafe" FFI calls passing an unlifted

94+-- pointer to the FFI call. For unsafe calls we do not need to pin the array.

95+-- We can pass an unlifted pointer to the FFI routine to avoid GC kicking in

96+-- before the pointer is wrapped.

97+--

98+-- From the GHC manual:

99+--

100+-- GHC, since version 8.4, guarantees that garbage collection will never occur

101+-- during an unsafe call, even in the bytecode interpreter, and further

102+-- guarantees that unsafe calls will be performed in the calling thread. Making

103+-- it safe to pass heap-allocated objects to unsafe functions.

104+

105+-- | Use a @MutByteArray@ as @Ptr a@. This is useful when we want to pass

106+-- an array as a pointer to some operating system call or to a "safe" FFI call.

107+--

108+-- If the array is not pinned it is copied to pinned memory before passing it

109+-- to the monadic action.

110+--

111+-- /Performance Notes:/ Forces a copy if the array is not pinned. It is advised

112+-- that the programmer keeps this in mind and creates a pinned array

113+-- opportunistically before this operation occurs, to avoid the cost of a copy

114+-- if possible.

115+--

116+-- /Unsafe/ because of direct pointer operations. The user must ensure that

117+-- they are writing within the legal bounds of the array.

118+--

119+-- /Pre-release/

120+--

121+{-# INLINE unsafePinnedAsPtr #-}

122+unsafePinnedAsPtr :: MonadIO m => MutByteArray -> (Ptr a -> m b) -> m b

123+unsafePinnedAsPtr arr f = do

124+ contents <- liftIO $ pin arr

125+ let !ptr = Ptr (byteArrayContents#

126+ (unsafeCoerce# (getMutableByteArray# contents)))

127+ r <- f ptr

128+ liftIO $ touch contents

129+ return r

130+

131+{-# DEPRECATED asPtrUnsafe "Please use unsafePinnedAsPtr instead." #-}

132+{-# INLINE asPtrUnsafe #-}

133+asPtrUnsafe :: MonadIO m => MutByteArray -> (Ptr a -> m b) -> m b

134+asPtrUnsafe = unsafePinnedAsPtr

135+

136+-- | For use with unsafe FFI functions. Does not force pin the array memory.

137+{-# INLINE unsafeAsPtr #-}

138+unsafeAsPtr :: MonadIO m => MutByteArray -> (Ptr a -> m b) -> m b

139+unsafeAsPtr arr f = do

140+ let !ptr = Ptr (byteArrayContents#

141+ (unsafeCoerce# (getMutableByteArray# arr)))

142+ r <- f ptr

143+ liftIO $ touch arr

144+ return r

145+

146+--------------------------------------------------------------------------------

147+-- Creation

148+--------------------------------------------------------------------------------

149+

150+{-# NOINLINE empty #-}

151+empty :: MutByteArray

152+empty = unsafePerformIO $ new 0

153+

154+{-# DEPRECATED nil "Please use empty instead" #-}

155+nil :: MutByteArray

156+nil = empty

157+

158+-- XXX add "newRounded" to round up the large size to the next page boundary

159+-- and return the allocated size.

160+{-# INLINE new #-}

161+new :: Int -> IO MutByteArray

162+new nbytes | nbytes < 0 =

163+ errorWithoutStackTrace "newByteArray: size must be >= 0"

164+new (I# nbytes) = IO $ \s ->

165+ case newByteArray# nbytes s of

166+ (# s', mbarr# #) ->

167+ let c = MutByteArray mbarr#

168+ in (# s', c #)

169+

170+{-# INLINE pinnedNew #-}

171+pinnedNew :: Int -> IO MutByteArray

172+pinnedNew nbytes | nbytes < 0 =

173+ errorWithoutStackTrace "pinnedNew: size must be >= 0"

174+pinnedNew (I# nbytes) = IO $ \s ->

175+ case newPinnedByteArray# nbytes s of

176+ (# s', mbarr# #) ->

177+ let c = MutByteArray mbarr#

178+ in (# s', c #)

179+

180+{-# INLINE pinnedNewAlignedBytes #-}

181+pinnedNewAlignedBytes :: Int -> Int -> IO MutByteArray

182+pinnedNewAlignedBytes nbytes _align | nbytes < 0 =

183+ errorWithoutStackTrace "pinnedNewAlignedBytes: size must be >= 0"

184+pinnedNewAlignedBytes (I# nbytes) (I# align) = IO $ \s ->

185+ case newAlignedPinnedByteArray# nbytes align s of

186+ (# s', mbarr# #) ->

187+ let c = MutByteArray mbarr#

188+ in (# s', c #)

189+

190+{-# INLINE newBytesAs #-}

191+newBytesAs :: PinnedState -> Int -> IO MutByteArray

192+newBytesAs Unpinned = new

193+newBytesAs Pinned = pinnedNew

194+

195+-------------------------------------------------------------------------------

196+-- Copying

197+-------------------------------------------------------------------------------

198+

199+-- Note: Array copy is more efficient than streaming copy.

200+-- CopyMutableByteArray# translates to genMemcpy in GHC/CmmToAsm/X86/CodeGen.hs

201+-- glibc memcpy copies bytes/words/pages - unrolls the loops:

202+-- https://github.com/bminor/glibc/blob/4290aed05135ae4c0272006442d147f2155e70d7/string/memcpy.c

203+-- https://github.com/bminor/glibc/blob/4290aed05135ae4c0272006442d147f2155e70d7/string/wordcopy.c

204+

205+-- | Put a sub range of a source array into a subrange of a destination array.

206+-- This is not safe as it does not check the bounds of neither the src array

207+-- nor the destination array.

208+{-# INLINE putSliceUnsafe #-}

209+putSliceUnsafe ::

210+ MonadIO m

211+ => MutByteArray

212+ -> Int

213+ -> MutByteArray

214+ -> Int

215+ -> Int

216+ -> m ()

217+putSliceUnsafe src srcStartBytes dst dstStartBytes lenBytes = liftIO $ do

218+#ifdef DEBUG

219+ srcLen <- sizeOfMutableByteArray src

220+ dstLen <- sizeOfMutableByteArray dst

221+ when (srcLen - srcStartBytes < lenBytes)

222+ $ error $ "putSliceUnsafe: src overflow: start" ++ show srcStartBytes

223+ ++ " end " ++ show srcLen ++ " len " ++ show lenBytes

224+ when (dstLen - dstStartBytes < lenBytes)

225+ $ error $ "putSliceUnsafe: dst overflow: start" ++ show dstStartBytes

226+ ++ " end " ++ show dstLen ++ " len " ++ show lenBytes

227+#endif

228+ let !(I# srcStartBytes#) = srcStartBytes

229+ !(I# dstStartBytes#) = dstStartBytes

230+ !(I# lenBytes#) = lenBytes

231+ let arrS# = getMutableByteArray# src

232+ arrD# = getMutableByteArray# dst

233+ IO $ \s# -> (# copyMutableByteArray#

234+ arrS# srcStartBytes# arrD# dstStartBytes# lenBytes# s#

235+ , () #)

236+

237+-- | Unsafe as it does not check whether the start offset and length supplied

238+-- are valid inside the array.

239+{-# INLINE cloneSliceUnsafeAs #-}

240+cloneSliceUnsafeAs :: MonadIO m =>

241+ PinnedState -> Int -> Int -> MutByteArray -> m MutByteArray

242+cloneSliceUnsafeAs ps srcOff srcLen src =

243+ liftIO $ do

244+ mba <- newBytesAs ps srcLen

245+ putSliceUnsafe src srcOff mba 0 srcLen

246+ return mba

247+

248+-- | @cloneSliceUnsafe offset len arr@ clones a slice of the supplied array

249+-- starting at the given offset and equal to the given length.

250+{-# INLINE cloneSliceUnsafe #-}

251+cloneSliceUnsafe :: MonadIO m => Int -> Int -> MutByteArray -> m MutByteArray

252+cloneSliceUnsafe = cloneSliceUnsafeAs Unpinned

253+

254+-- | @pinnedCloneSliceUnsafe offset len arr@

255+{-# INLINE pinnedCloneSliceUnsafe #-}

256+pinnedCloneSliceUnsafe :: MonadIO m =>

257+ Int -> Int -> MutByteArray -> m MutByteArray

258+pinnedCloneSliceUnsafe = cloneSliceUnsafeAs Pinned

259+

260+-------------------------------------------------------------------------------

261+-- Pinning & Unpinning

262+-------------------------------------------------------------------------------

263+

264+-- | Return 'True' if the array is allocated in pinned memory.

265+{-# INLINE isPinned #-}

266+isPinned :: MutByteArray -> Bool

267+isPinned (MutByteArray arr#) =

268+ let pinnedInt = I# (isMutableByteArrayPinned# arr#)

269+ in pinnedInt /= 0

270+

271+

272+{-# INLINE cloneMutableArrayWith# #-}

273+cloneMutableArrayWith#

274+ :: (Int# -> State# RealWorld -> (# State# RealWorld

275+ , MutableByteArray# RealWorld #))

276+ -> MutableByteArray# RealWorld

277+ -> State# RealWorld

278+ -> (# State# RealWorld, MutableByteArray# RealWorld #)

279+cloneMutableArrayWith# alloc# arr# s# =

280+ case getSizeofMutableByteArray# arr# s# of

281+ (# s1#, i# #) ->

282+ case alloc# i# s1# of

283+ (# s2#, arr1# #) ->

284+ case copyMutableByteArray# arr# 0# arr1# 0# i# s2# of

285+ s3# -> (# s3#, arr1# #)

286+

287+-- | Return a copy of the array in pinned memory if unpinned, else return the

288+-- original array.

289+{-# INLINE pin #-}

290+pin :: MutByteArray -> IO MutByteArray

291+pin arr@(MutByteArray marr#) =

292+ if isPinned arr

293+ then return arr

294+ else

295+#ifdef DEBUG

296+ do

297+ -- XXX dump stack trace

298+ trace ("pin: Copying array") (return ())

299+#endif

300+ IO

301+ $ \s# ->

302+ case cloneMutableArrayWith# newPinnedByteArray# marr# s# of

303+ (# s1#, marr1# #) -> (# s1#, MutByteArray marr1# #)

304+

305+-- | Return a copy of the array in unpinned memory if pinned, else return the

306+-- original array.

307+{-# INLINE unpin #-}

308+unpin :: MutByteArray -> IO MutByteArray

309+unpin arr@(MutByteArray marr#) =

310+ if not (isPinned arr)

311+ then return arr

312+ else

313+#ifdef DEBUG

314+ do

315+ -- XXX dump stack trace

316+ trace ("unpin: Copying array") (return ())

317+#endif

318+ IO

319+ $ \s# ->

320+ case cloneMutableArrayWith# newByteArray# marr# s# of

321+ (# s1#, marr1# #) -> (# s1#, MutByteArray marr1# #)

core/src/Streamly/Internal/Data/Parser.hs+3

@@ -10,6 +10,9 @@

1010module Streamly.Internal.Data.Parser

1111 (

1212 -- * Setup

13+ -- | To execute the code examples provided in this module in ghci, please

14+ -- run the following commands first.

15+ --

1316 -- $setup

1417

1518 module Streamly.Internal.Data.Parser.Type

core/src/Streamly/Internal/Data/Parser/Type.hs+72−52

@@ -175,9 +175,6 @@

175175

176176module Streamly.Internal.Data.Parser.Type

177177 (

178− -- * Setup

179− -- $setup

180−

181178 -- * Types

182179 Initial (..)

183180 , Step (..)

@@ -441,6 +438,7 @@ newtype ParseError = ParseError String

441438instance Exception ParseError where

442439 displayException (ParseError err) = err

443440

441+-- | Map a function on the result i.e. on @b@ in @Parser a m b@.

444442instance Functor m => Functor (Parser a m) where

445443 {-# INLINE fmap #-}

446444 fmap f (Parser step1 initial1 extract) =

@@ -507,26 +505,30 @@ data SeqParseState sl f sr = SeqParseL !sl | SeqParseR !f !sr

507505-- 'Streamly.Data.Stream.append', it splits the streams using two parsers and

508506-- zips the results.

509507

510−-- | Sequential parser application.

508+-- | Sequential parser application. Apply two parsers sequentially to an input

509+-- stream. The first parser runs and processes the input, the remaining input

510+-- is then passed to the second parser. If both parsers succeed, their outputs

511+-- are combined using the supplied function. If either parser fails, the

512+-- operation fails.

511513--

512−-- Apply two parsers sequentially to an input stream. The first parser runs and

513−-- processes the input, the remaining input is then passed to the second

514−-- parser. If both parsers succeed, their outputs are combined using the

515−-- supplied function. If either parser fails, the operation fails.

514+-- This combinator delivers high performance by stream fusion but it comes with

515+-- some limitations. For those cases use the 'Applicative' instance of

516+-- 'Streamly.Data.ParserK.ParserK'.

516517--

517−-- This implementation is strict in the second argument, therefore, the

518−-- following will fail:

518+-- CAVEAT 1: NO RECURSION. This function is strict in both arguments. As a

519+-- result, if a parser is defined recursively using this, it may cause an

520+-- infintie loop. The following example checks the strictness:

519521--

520−-- >>> Stream.parse (Parser.splitWith const (Parser.satisfy (> 0)) undefined) $ Stream.fromList [1]

522+-- >>> p = Parser.splitWith const (Parser.satisfy (> 0)) undefined

523+-- >>> Stream.parse p $ Stream.fromList [1]

521524-- *** Exception: Prelude.undefined

522525-- ...

523526--

524−-- Although this implementation allows stream fusion, it has quadratic

525−-- complexity, making it suitable only for a small number of compositions.

526−-- As a thumb rule use it for less than 8 compositions, use ParserK otherwise.

527+-- CAVEAT 2: QUADRATIC TIME COMPLEXITY. Static composition is fast due to

528+-- stream fusion, but it works well only for limited (e.g. up to 8)

529+-- compositions, use "Streamly.Data.ParserK" for larger compositions.

527530--

528−-- Below are some common idioms that can be expressed using 'splitWith' and

529−-- other parser primitives:

531+-- Below are some common idioms that can be expressed using 'splitWith':

530532--

531533-- >>> span p f1 f2 = Parser.splitWith (,) (Parser.takeWhile p f1) (Parser.fromFold f2)

532534-- >>> spanBy eq f1 f2 = Parser.splitWith (,) (Parser.groupBy eq f1) (Parser.fromFold f2)

@@ -615,7 +617,12 @@ splitWith func (Parser stepL initialL extractL)

615617-- We can use the compiler to automatically annotate accumulators, terminating

616618-- folds, non-failing parsers and failing parsers.

617619

618−-- | Works correctly only if both the parsers are guaranteed to never fail.

620+-- | Better performance 'splitWith' for non-failing parsers.

621+--

622+-- Does not work correctly for parsers that can fail.

623+--

624+-- ALL THE CAVEATS IN 'splitWith' APPLY HERE AS WELL.

625+--

619626{-# INLINE noErrorUnsafeSplitWith #-}

620627noErrorUnsafeSplitWith :: Monad m

621628 => (a -> b -> c) -> Parser x m a -> Parser x m b -> Parser x m c

@@ -686,6 +693,8 @@ data SeqAState sl sr = SeqAL !sl | SeqAR !sr

686693-- second parser. The output of the parser is the output of the second parser.

687694-- The operation fails if any of the parsers fail.

688695--

696+-- ALL THE CAVEATS IN 'splitWith' APPLY HERE AS WELL.

697+--

689698-- This implementation is strict in the second argument, therefore, the

690699-- following will fail:

691700--

@@ -693,10 +702,6 @@ data SeqAState sl sr = SeqAL !sl | SeqAR !sr

693702-- *** Exception: Prelude.undefined

694703-- ...

695704--

696−-- Although this implementation allows stream fusion, it has quadratic

697−-- complexity, making it suitable only for a small number of compositions.

698−-- As a thumb rule use it for less than 8 compositions, use ParserK otherwise.

699−--

700705-- /Pre-release/

701706--

702707{-# INLINE split_ #-}

@@ -752,7 +757,12 @@ split_ (Parser stepL initialL extractL) (Parser stepR initialR extractR) =

752757 Partial _ _ -> error "split_: Partial"

753758 Continue n s -> return $ Continue n (SeqAL s)

754759

755−-- For backtracking folds

760+-- | Better performance 'split_' for non-failing parsers.

761+--

762+-- Does not work correctly for parsers that can fail.

763+--

764+-- ALL THE CAVEATS IN 'splitWith' APPLY HERE AS WELL.

765+--

756766{-# INLINE noErrorUnsafeSplit_ #-}

757767noErrorUnsafeSplit_ :: Monad m => Parser x m a -> Parser x m b -> Parser x m b

758768noErrorUnsafeSplit_

@@ -807,7 +817,11 @@ noErrorUnsafeSplit_

807817 Partial _ _ -> error "split_: Partial"

808818 Continue n s -> return $ Continue n (SeqAL s)

809819

810−-- | 'Applicative' form of 'splitWith'.

820+-- | READ THE CAVEATS in 'splitWith' before using this instance.

821+--

822+-- >>> pure = Parser.fromPure

823+-- >>> (<*>) = Parser.splitWith id

824+-- >>> (*>) = Parser.split_

811825instance Monad m => Applicative (Parser a m) where

812826 {-# INLINE pure #-}

813827 pure = fromPure

@@ -833,20 +847,26 @@ data AltParseState sl sr = AltParseL !Int !sl | AltParseR !sr

833847-- each subsequent alternative's input element has to go through, therefore, it

834848-- cannot scale to a large number of compositions

835849

836−-- | Sequential alternative. The input is first passed to the first parser, and

850+-- | Sequential alternative. The input is first passed to the first parser,

837851-- if it succeeds, the result is returned. However, if the first parser fails,

838−-- the parser driver backtracks and tries the same input on the second parser,

839−-- returning the result if it succeeds.

852+-- the parser driver backtracks and tries the same input on the second

853+-- (alternative) parser, returning the result if it succeeds.

854+--

855+-- This combinator delivers high performance by stream fusion but it comes with

856+-- some limitations. For those cases use the 'Alternative' instance of

857+-- 'Streamly.Data.ParserK.ParserK'.

840858--

841−-- Note: This implementation is not lazy in the second argument. The following

842−-- will fail:

859+-- CAVEAT 1: NO RECURSION. This function is strict in both arguments. As a

860+-- result, if a parser is defined recursively using this, it may cause an

861+-- infintie loop. The following example checks the strictness:

843862--

844−-- >> Stream.parse (Parser.satisfy (> 0) `Parser.alt` undefined) $ Stream.fromList [1..10]

863+-- >>> p = Parser.satisfy (> 0) `Parser.alt` undefined

864+-- >>> Stream.parse p $ Stream.fromList [1..10]

845865-- *** Exception: Prelude.undefined

846866--

847−-- Although this implementation allows stream fusion, it has quadratic

848−-- complexity, making it suitable only for a small number of compositions.

849−-- As a thumb rule use it for less than 8 compositions, use ParserK otherwise.

867+-- CAVEAT 2: QUADRATIC TIME COMPLEXITY. Static composition is fast due to

868+-- stream fusion, but it works well only for limited (e.g. up to 8)

869+-- compositions, use "Streamly.Data.ParserK" for larger compositions.

850870--

851871-- /Time Complexity:/ O(n^2) where n is the number of compositions.

852872--

@@ -1176,23 +1196,12 @@ dieM err = Parser undefined (IError <$> err) undefined

11761196-- Note: With the direct style parser type, the list in "some" and "many" is

11771197-- accumulated strictly, it cannot be consumed lazily.

11781198

1179−-- | Sequential alternative. The input is first passed to the first parser, and

1180−-- if it succeeds, the result is returned. However, if the first parser fails,

1181−-- the parser driver backtracks and tries the same input on the second parser,

1182−-- returning the result if it succeeds.

1183−--

1184−-- Note: The implementation of '<|>' is not lazy in the second

1185−-- argument. The following code will fail:

1186−--

1187−-- >>> Stream.parse (Parser.satisfy (> 0) <|> undefined) $ Stream.fromList [1..10]

1188−-- *** Exception: Prelude.undefined

1189−-- ...

1199+-- | READ THE CAVEATS in 'alt' before using this instance.

11901200--

1191−-- WARNING! this is not suitable for large scale use. As a thumb rule stream

1192−-- fusion works well for less than 8 compositions of this operation, otherwise

1193−-- consider using 'ParserK'. Do not use recursive parser implementations based

1194−-- on this Alternative instance.

1195−

1201+-- >>> empty = Parser.die "empty"

1202+-- >>> (<|>) = Parser.alt

1203+-- >>> many = flip Parser.many Fold.toList

1204+-- >>> some = flip Parser.some Fold.toList

11961205instance Monad m => Alternative (Parser a m) where

11971206 {-# INLINE empty #-}

11981207 empty = die "empty"

@@ -1211,8 +1220,13 @@ data ConcatParseState sl m a b =

12111220 ConcatParseL !sl

12121221 | forall s. ConcatParseR (s -> a -> m (Step s b)) s (s -> m (Step s b))

12131222

1223+-- XXX Does it fuse completely? Check and update, it cannot fuse the

1224+-- dynamically generated parser.

1225+

12141226-- | Map a 'Parser' returning function on the result of a 'Parser'.

12151227--

1228+-- ALL THE CAVEATS IN 'splitWith' APPLY HERE AS WELL.

1229+--

12161230-- /Pre-release/

12171231--

12181232{-# INLINE concatMap #-}

@@ -1282,6 +1296,12 @@ concatMap func (Parser stepL initialL extractL) = Parser step initial extract

12821296 Partial _ _ -> error "concatMap: extract Partial"

12831297 Continue n s -> return $ Continue n (ConcatParseL s)

12841298

1299+-- | Better performance 'concatMap' for non-failing parsers.

1300+--

1301+-- Does not work correctly for parsers that can fail.

1302+--

1303+-- ALL THE CAVEATS IN 'splitWith' APPLY HERE AS WELL.

1304+--

12851305{-# INLINE noErrorUnsafeConcatMap #-}

12861306noErrorUnsafeConcatMap :: Monad m =>

12871307 (b -> Parser a m c) -> Parser a m b -> Parser a m c

@@ -1354,11 +1374,9 @@ noErrorUnsafeConcatMap func (Parser stepL initialL extractL) =

13541374-- for small number of compositions but for a scalable implementation we need a

13551375-- CPS version.

13561376

1357−-- | See documentation of 'Streamly.Internal.Data.Parser.ParserK.Type.Parser'.

1377+-- | READ THE CAVEATS in 'concatMap' before using this instance.

13581378--

1359−-- Although this implementation allows stream fusion, it has quadratic

1360−-- complexity, making it suitable only for a small number of compositions. As a

1361−-- thumb rule use it for less than 8 compositions, use 'ParserK' otherwise.

1379+-- >>> (>>=) = flip Parser.concatMap

13621380--

13631381instance Monad m => Monad (Parser a m) where

13641382 {-# INLINE return #-}

@@ -1370,6 +1388,7 @@ instance Monad m => Monad (Parser a m) where

13701388 {-# INLINE (>>) #-}

13711389 (>>) = (*>)

13721390

1391+-- | >>> fail = Parser.die

13731392instance Monad m => Fail.MonadFail (Parser a m) where

13741393 {-# INLINE fail #-}

13751394 fail = die

@@ -1385,6 +1404,7 @@ instance Monad m => MonadPlus (Parser a m) where

13851404 mplus = alt

13861405-}

13871406

1407+-- | >>> liftIO = Parser.fromEffect . liftIO

13881408instance (MonadIO m) => MonadIO (Parser a m) where

13891409 {-# INLINE liftIO #-}

13901410 liftIO = fromEffect . liftIO

core/src/Streamly/Internal/Data/ParserK/Type.hs+24−64

@@ -168,8 +168,8 @@ newtype ParserK a m b = MkParser

168168-------------------------------------------------------------------------------

169169

170170-- XXX rewrite this using ParserD, expose rmapM from ParserD.

171−-- | Maps a function over the output of the parser.

172−--

171+

172+-- | Map a function on the result i.e. on @b@ in @Parser a m b@.

173173instance Functor m => Functor (ParserK a m) where

174174 {-# INLINE fmap #-}

175175 fmap f parser = MkParser $ \k n st arr ->

@@ -181,7 +181,7 @@ instance Functor m => Functor (ParserK a m) where

181181-------------------------------------------------------------------------------

182182

183183-- This is the dual of stream "fromPure".

184−--

184+

185185-- | A parser that always yields a pure value without consuming any input.

186186--

187187-- /Pre-release/

@@ -199,9 +199,11 @@ fromEffect :: Monad m => m b -> ParserK a m b

199199fromEffect eff =

200200 MkParser $ \k n st arr -> eff >>= \b -> k (Success n b) st arr

201201

202−-- | 'Applicative' form of 'Streamly.Internal.Data.Parser.splitWith'. Note that

203−-- this operation does not fuse, use 'Streamly.Internal.Data.Parser.splitWith'

204−-- when fusion is important.

202+-- | @f \<$> p1 \<*> p2@ applies parsers p1 and p2 sequentially to an input

203+-- stream. The first parser runs and processes the input, the remaining input

204+-- is then passed to the second parser. If both parsers succeed, their outputs

205+-- are applied to the function @f@. If either parser fails, the operation

206+-- fails.

205207--

206208instance Monad m => Applicative (ParserK a m) where

207209 {-# INLINE pure #-}

@@ -243,48 +245,8 @@ instance Monad m => Applicative (ParserK a m) where

243245die :: String -> ParserK a m b

244246die err = MkParser (\k n st arr -> k (Failure n err) st arr)

245247

246−-- | Monad composition can be used for lookbehind parsers, we can make the

247−-- future parses depend on the previously parsed values.

248−--

249−-- If we have to parse "a9" or "9a" but not "99" or "aa" we can use the

250−-- following parser:

251−--

252−-- @

253−-- backtracking :: MonadCatch m => PR.Parser Char m String

254−-- backtracking =

255−-- sequence [PR.satisfy isDigit, PR.satisfy isAlpha]

256−-- '<|>'

257−-- sequence [PR.satisfy isAlpha, PR.satisfy isDigit]

258−-- @

259−--

260−-- We know that if the first parse resulted in a digit at the first place then

261−-- the second parse is going to fail. However, we waste that information and

262−-- parse the first character again in the second parse only to know that it is

263−-- not an alphabetic char. By using lookbehind in a 'Monad' composition we can

264−-- avoid redundant work:

265−--

266−-- @

267−-- data DigitOrAlpha = Digit Char | Alpha Char

268−--

269−-- lookbehind :: MonadCatch m => PR.Parser Char m String

270−-- lookbehind = do

271−-- x1 \<- Digit '<$>' PR.satisfy isDigit

272−-- '<|>' Alpha '<$>' PR.satisfy isAlpha

273−--

274−-- -- Note: the parse depends on what we parsed already

275−-- x2 <- case x1 of

276−-- Digit _ -> PR.satisfy isAlpha

277−-- Alpha _ -> PR.satisfy isDigit

278−--

279−-- return $ case x1 of

280−-- Digit x -> [x,x2]

281−-- Alpha x -> [x,x2]

282−-- @

283−--

284−-- See also 'Streamly.Internal.Data.Parser.concatMap'. This monad instance

285−-- does not fuse, use 'Streamly.Internal.Data.Parser.concatMap' when you need

286−-- fusion.

287−--

248+-- | Monad composition can be used for lookbehind parsers, we can dynamically

249+-- compose new parsers based on the results of the previously parsed values.

288250instance Monad m => Monad (ParserK a m) where

289251 {-# INLINE return #-}

290252 return = pure

@@ -316,18 +278,10 @@ instance MonadIO m => MonadIO (ParserK a m) where

316278-- Alternative

317279-------------------------------------------------------------------------------

318280

319−-- | 'Alternative' form of 'Streamly.Internal.Data.Parser.alt'. Backtrack and

320−-- run the second parser if the first one fails.

321−--

322−-- The "some" and "many" operations of alternative accumulate results in a pure

323−-- list which is not scalable and streaming. Instead use

324−-- 'Streamly.Internal.Data.Parser.some' and

325−-- 'Streamly.Internal.Data.Parser.many' for fusible operations with composable

326−-- accumulation of results.

327−--

328−-- See also 'Streamly.Internal.Data.Parser.alt'. This 'Alternative' instance

329−-- does not fuse, use 'Streamly.Internal.Data.Parser.alt' when you need

330−-- fusion.

281+-- | @p1 \<|> p2@ passes the input to parser p1, if it succeeds, the result is

282+-- returned. However, if p1 fails, the parser driver backtracks and tries the

283+-- same input on the alternative parser p2, returning the result if it

284+-- succeeds.

331285--

332286instance Monad m => Alternative (ParserK a m) where

333287 {-# INLINE empty #-}

@@ -450,7 +404,7 @@ adaptCWith pstep initial extract cont !offset0 !usedCount !input = do

450404 go !_ !cur !pst | cur >= end =

451405 onContinue ((end - start) `div` SIZE_OF(a)) pst

452406 go !_ !cur !pst = do

453− let !x = unsafeInlineIO $ peekByteIndex cur contents

407+ let !x = unsafeInlineIO $ peekAt cur contents

454408 pRes <- pstep pst x

455409 let elemSize = SIZE_OF(a)

456410 next = INDEX_NEXT(cur,a)

@@ -507,7 +461,8 @@ adaptCWith pstep initial extract cont !offset0 !usedCount !input = do

507461 parseCont !cnt !pst (Chunk arr) = parseContChunk cnt 0 pst arr

508462 parseCont !cnt !pst None = parseContNothing cnt pst

509463

510−-- | Convert a raw byte 'Parser' to a chunked 'ParserK'.

464+-- | Convert an element 'Parser' to a chunked 'ParserK'. A chunked parser is

465+-- more efficient than an element parser.

511466--

512467-- /Pre-release/

513468--

@@ -609,7 +564,9 @@ adaptWith pstep initial extract cont !relPos !usedCount !input = do

609564 parseCont !cnt !pst (Chunk arr) = parseContChunk cnt pst arr

610565 parseCont !cnt !pst None = parseContNothing cnt pst

611566

612−-- | Similar to "adaptC" but for non-chunked input.

567+-- | Convert a 'Parser' to 'ParserK'.

568+--

569+-- /Pre-release/

613570--

614571{-# INLINE_LATE adapt #-}

615572adapt :: Monad m => ParserD.Parser a m b -> ParserK a m b

@@ -737,7 +694,10 @@ adaptCGWith pstep initial extract cont !offset0 !usedCount !input = do

737694 parseCont !cnt !pst (Chunk arr) = parseContChunk cnt 0 pst arr

738695 parseCont !cnt !pst None = parseContNothing cnt pst

739696

740−-- | Similar to "adaptC" but is not constrained.

697+-- | A generic 'adaptC'. Similar to 'adaptC' but is not constrained to 'Unbox'

698+-- types.

699+--

700+-- /Pre-release/

741701--

742702{-# INLINE_LATE adaptCG #-}

743703adaptCG ::

core/src/Streamly/Internal/Data/Ring.hs+3−3

@@ -79,7 +79,7 @@ import Foreign.Storable

7979import Foreign.ForeignPtr (ForeignPtr, withForeignPtr, touchForeignPtr)

8080import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)

8181import Foreign.Ptr (plusPtr, minusPtr, castPtr)

82−import Streamly.Internal.Data.Unbox as Unboxed (Unbox(peekByteIndex))

82+import Streamly.Internal.Data.Unbox as Unboxed (Unbox(peekAt))

8383import GHC.ForeignPtr (mallocPlainForeignPtrAlignedBytes)

8484import GHC.Ptr (Ptr(..))

8585import Streamly.Internal.Data.MutArray.Type (MutArray)

@@ -411,7 +411,7 @@ unsafeEqArrayN Ring{..} rh A.Array{..} nBytes

411411

412412 check p i = do

413413 (relem :: Word8) <- peek p

414− aelem <- peekByteIndex i w8Contents

414+ aelem <- peekAt i w8Contents

415415 if relem == aelem

416416 then go (p `plusPtr` 1) (i + 1)

417417 else return False

@@ -445,7 +445,7 @@ unsafeEqArray Ring{..} rh A.Array{..} =

445445

446446 check p i = do

447447 (relem :: Word8) <- peek p

448− aelem <- peekByteIndex i w8Contents

448+ aelem <- peekAt i w8Contents

449449 if relem == aelem

450450 then go (p `plusPtr` 1) (i + 1)

451451 else return False

core/src/Streamly/Internal/Data/Serialize/TH.hs+113−96

@@ -7,12 +7,23 @@

77-- Maintainer : streamly@composewell.com

88-- Stability : experimental

99-- Portability : GHC

10−--

10+

11+-- XXX Instead of applying the product constructor in one go can we apply it

12+-- one at a time in case of too many arguments, compiler may not have to save

13+-- them in local vars.

14+

1115module Streamly.Internal.Data.Serialize.TH

12− ( deriveSerialize

13− , Config(..)

14− , defaultConfig

16+ (

17+ -- Deriving

18+ deriveSerialize

1519 , deriveSerializeWith

20+

21+ -- Utilities

22+ , module Streamly.Internal.Data.Serialize.TH.Bottom

23+ -- ** Common

24+ , module Streamly.Internal.Data.Serialize.TH.Common

25+ -- ** RecHeader

26+ , module Streamly.Internal.Data.Serialize.TH.RecHeader

1627 ) where

1728

1829--------------------------------------------------------------------------------

@@ -29,7 +40,6 @@ import Streamly.Internal.Data.Serialize.Type

2940import Streamly.Internal.Data.Unbox.TH

3041 ( DataCon(..)

3142 , DataType(..)

32− , appsT

3343 , reifyDataType

3444 )

3545

@@ -37,13 +47,14 @@ import qualified Streamly.Internal.Data.Serialize.TH.RecHeader as RecHeader

3747

3848import Streamly.Internal.Data.Serialize.TH.Bottom

3949import Streamly.Internal.Data.Serialize.TH.Common

50+import Streamly.Internal.Data.Serialize.TH.RecHeader

4051

4152--------------------------------------------------------------------------------

4253-- Domain specific helpers

4354--------------------------------------------------------------------------------

4455

4556exprGetSize :: Q Exp -> (Int, Type) -> Q Exp

46−exprGetSize acc (i, _) = [|size $(acc) $(varE (mkFieldName i))|]

57+exprGetSize acc (i, _) = [|addSizeTo $(acc) $(varE (mkFieldName i))|]

4758

4859getTagSize :: Int -> Int

4960getTagSize numConstructors

@@ -148,22 +159,25 @@ mkSizeOfExpr False False tyOfTy =

148159 [varP _acc, varP _x]

149160 (caseE (varE _x) (fmap (matchCons acc) cons))

150161

151−mkSizeOfExpr False True (TheType con) = RecHeader.mkSizeOfExpr con

162+mkSizeOfExpr False True (TheType con) = RecHeader.mkRecSizeOfExpr con

152163

153164mkSizeOfExpr _ _ _ = errorUnimplemented

154165

155−mkSizeDec :: Config -> Type -> [DataCon] -> Q [Dec]

156−mkSizeDec (Config {..}) headTy cons = do

166+mkSizeDec :: SerializeConfig -> Type -> [DataCon] -> Q [Dec]

167+mkSizeDec (SerializeConfig {..}) headTy cons = do

157168 -- INLINE on sizeOf actually worsens some benchmarks, and improves none

158169 sizeOfMethod <-

159170 mkSizeOfExpr

160− constructorTagAsString

161− recordSyntaxWithHeader

171+ cfgConstructorTagAsString

172+ cfgRecordSyntaxWithHeader

162173 (typeOfType headTy cons)

163174 pure

164− [ PragmaD (InlineP 'size inlineSize FunLike AllPhases)

165− , FunD 'size [Clause [] (NormalB sizeOfMethod) []]

166− ]

175+ ( maybe

176+ []

177+ (\x -> [PragmaD (InlineP 'addSizeTo x FunLike AllPhases)])

178+ cfgInlineSize

179+ ++ [FunD 'addSizeTo [Clause [] (NormalB sizeOfMethod) []]]

180+ )

167181

168182--------------------------------------------------------------------------------

169183-- Peek

@@ -182,7 +196,7 @@ mkDeserializeExpr True False headTy tyOfTy =

182196 conLen <- newName "conLen"

183197 off1 <- newName "off1"

184198 [|do ($(varP off1), $(varP conLen) :: Word8) <-

185− deserialize

199+ deserializeAt

186200 $(varE _initialOffset)

187201 $(varE _arr)

188202 $(varE _endOffset)

@@ -202,18 +216,19 @@ mkDeserializeExpr True False headTy tyOfTy =

202216 [|($(litIntegral lenCname) == $(varE conLen))

203217 && $(xorCmp tag off _arr)|]

204218 [|let $(varP (makeI 0)) = $(varE off) + $(litIntegral lenCname)

205− in $(mkDeserializeExprOne 'deserialize con)|]

219+ in $(mkDeserializeExprOne 'deserializeAt con)|]

206220

207221mkDeserializeExpr False False headTy tyOfTy =

208222 case tyOfTy of

209223 -- Unit constructor

224+ -- XXX Should we peek and check if the byte value is 0?

210225 UnitType cname ->

211226 [|pure ($(varE _initialOffset) + 1, $(conE cname))|]

212227 -- Product type

213228 TheType con ->

214229 letE

215230 [valD (varP (mkName "i0")) (normalB (varE _initialOffset)) []]

216− (mkDeserializeExprOne 'deserialize con)

231+ (mkDeserializeExprOne 'deserializeAt con)

217232 -- Sum type

218233 MultiType cons -> do

219234 let lenCons = length cons

@@ -221,7 +236,7 @@ mkDeserializeExpr False False headTy tyOfTy =

221236 doE

222237 [ bindS

223238 (tupP [varP (mkName "i0"), varP _tag])

224− [|deserialize $(varE _initialOffset) $(varE _arr) $(varE _endOffset)|]

239+ [|deserializeAt $(varE _initialOffset) $(varE _arr) $(varE _endOffset)|]

225240 , noBindS

226241 (caseE

227242 (sigE (varE _tag) (conT tagType))

@@ -231,7 +246,7 @@ mkDeserializeExpr False False headTy tyOfTy =

231246 peekMatch (i, con) =

232247 match

233248 (litP (IntegerL i))

234− (normalB (mkDeserializeExprOne 'deserialize con)) []

249+ (normalB (mkDeserializeExprOne 'deserializeAt con)) []

235250 peekErr =

236251 match

237252 wildP

@@ -250,7 +265,7 @@ mkDeserializeExpr False True _ (TheType con@(SimpleDataCon _ fields)) = do

250265 RecHeader.mkDeserializeKeysDec deserializeWithKeys updateFunc con

251266 letE

252267 (pure <$> (deserializeWithKeysDec ++ updateFuncDec))

253− (RecHeader.mkDeserializeExpr

268+ (RecHeader.mkRecDeserializeExpr

254269 _initialOffset

255270 _endOffset

256271 deserializeWithKeys

@@ -258,26 +273,31 @@ mkDeserializeExpr False True _ (TheType con@(SimpleDataCon _ fields)) = do

258273

259274mkDeserializeExpr _ _ _ _ = errorUnimplemented

260275

261−mkDeserializeDec :: Config -> Type -> [DataCon] -> Q [Dec]

262−mkDeserializeDec (Config {..}) headTy cons = do

276+mkDeserializeDec :: SerializeConfig -> Type -> [DataCon] -> Q [Dec]

277+mkDeserializeDec (SerializeConfig {..}) headTy cons = do

263278 peekMethod <-

264279 mkDeserializeExpr

265− constructorTagAsString

266− recordSyntaxWithHeader

280+ cfgConstructorTagAsString

281+ cfgRecordSyntaxWithHeader

267282 headTy

268283 (typeOfType headTy cons)

269284 pure

270− [ PragmaD (InlineP 'deserialize inlineDeserialize FunLike AllPhases)

271− , FunD

272− 'deserialize

285+ ( maybe

286+ []

287+ (\x -> [PragmaD (InlineP 'deserializeAt x FunLike AllPhases)])

288+ cfgInlineDeserialize

289+ ++

290+ [ FunD

291+ 'deserializeAt

273292 [ Clause

274− (if isUnitType cons && not constructorTagAsString

293+ (if isUnitType cons && not cfgConstructorTagAsString

275294 then [VarP _initialOffset, WildP, WildP]

276295 else [VarP _initialOffset, VarP _arr, VarP _endOffset])

277296 (NormalB peekMethod)

278297 []

279298 ]

280− ]

299+ ]

300+ )

281301

282302--------------------------------------------------------------------------------

283303-- Poke

@@ -285,7 +305,7 @@ mkDeserializeDec (Config {..}) headTy cons = do

285305

286306mkSerializeExprTag :: Name -> Int -> Q Exp

287307mkSerializeExprTag tagType tagVal =

288− [|serialize

308+ [|serializeAt

289309 $(varE _initialOffset)

290310 $(varE _arr)

291311 $((sigE (litE (IntegerL (fromIntegral tagVal))) (conT tagType)))|]

@@ -320,13 +340,14 @@ mkSerializeExpr True False tyOfTy =

320340 (doE [ bindS

321341 (varP (mkName "i0"))

322342 (serializeW8List _initialOffset _arr conEnc)

323− , noBindS (mkSerializeExprFields 'serialize fields)

343+ , noBindS (mkSerializeExprFields 'serializeAt fields)

324344 ])

325345

326346mkSerializeExpr False False tyOfTy =

327347 case tyOfTy of

328348 -- Unit type

329− UnitType _ -> [|pure ($(varE _initialOffset) + 1)|]

349+ UnitType _ ->

350+ [|serializeAt $(varE _initialOffset) $(varE _arr) (0 :: Word8)|]

330351 -- Product type

331352 (TheType (SimpleDataCon cname fields)) ->

332353 letE

@@ -336,7 +357,7 @@ mkSerializeExpr False False tyOfTy =

336357 [ matchConstructor

337358 cname

338359 (length fields)

339− (mkSerializeExprFields 'serialize fields)

360+ (mkSerializeExprFields 'serializeAt fields)

340361 ])

341362 -- Sum type

342363 (MultiType cons) -> do

@@ -353,35 +374,40 @@ mkSerializeExpr False False tyOfTy =

353374 (mkSerializeExprTag tagType tagVal)

354375 , noBindS

355376 (mkSerializeExprFields

356− 'serialize

377+ 'serializeAt

357378 fields)

358379 ]))

359380 (zip [0 ..] cons))

360381

361382mkSerializeExpr False True (TheType con) =

362− RecHeader.mkSerializeExpr _initialOffset con

383+ RecHeader.mkRecSerializeExpr _initialOffset con

363384

364385mkSerializeExpr _ _ _ = errorUnimplemented

365386

366−mkSerializeDec :: Config -> Type -> [DataCon] -> Q [Dec]

367−mkSerializeDec (Config {..}) headTy cons = do

387+mkSerializeDec :: SerializeConfig -> Type -> [DataCon] -> Q [Dec]

388+mkSerializeDec (SerializeConfig {..}) headTy cons = do

368389 pokeMethod <-

369390 mkSerializeExpr

370− constructorTagAsString

371− recordSyntaxWithHeader

391+ cfgConstructorTagAsString

392+ cfgRecordSyntaxWithHeader

372393 (typeOfType headTy cons)

373394 pure

374− [ PragmaD (InlineP 'serialize inlineSerialize FunLike AllPhases)

375− , FunD

376− 'serialize

377− [ Clause

378− (if isUnitType cons && not constructorTagAsString

379− then [VarP _initialOffset, WildP, WildP]

380− else [VarP _initialOffset, VarP _arr, VarP _val])

381− (NormalB pokeMethod)

382− []

383− ]

384− ]

395+ ( maybe

396+ []

397+ (\x -> [PragmaD (InlineP 'serializeAt x FunLike AllPhases)])

398+ cfgInlineSerialize

399+ ++

400+ [FunD

401+ 'serializeAt

402+ [ Clause

403+ (if isUnitType cons && not cfgConstructorTagAsString

404+ then [VarP _initialOffset, VarP _arr, WildP]

405+ else [VarP _initialOffset, VarP _arr, VarP _val])

406+ (NormalB pokeMethod)

407+ []

408+ ]

409+ ]

410+ )

385411

386412--------------------------------------------------------------------------------

387413-- Main

@@ -403,7 +429,7 @@ mkSerializeDec (Config {..}) headTy cons = do

403429-- Usage:

404430-- @

405431-- $(deriveSerializeInternal

406−-- defaultConfig

432+-- serializeConfig

407433-- [AppT (ConT ''Serialize) (VarT (mkName "b"))]

408434-- (AppT

409435-- (AppT (ConT ''CustomDataType) (VarT (mkName "a")))

@@ -418,7 +444,7 @@ mkSerializeDec (Config {..}) headTy cons = do

418444-- ])

419445-- @

420446deriveSerializeInternal ::

421− Config -> Type -> [DataCon] -> ([Dec] -> Q [Dec]) -> Q [Dec]

447+ SerializeConfig -> Type -> [DataCon] -> ([Dec] -> Q [Dec]) -> Q [Dec]

422448deriveSerializeInternal conf headTy cons next = do

423449 sizeDec <- mkSizeDec conf headTy cons

424450 peekDec <- mkDeserializeDec conf headTy cons

@@ -426,30 +452,41 @@ deriveSerializeInternal conf headTy cons next = do

426452 let methods = concat [sizeDec, peekDec, pokeDec]

427453 next methods

428454

429−-- | Similar to 'deriveSerialize' but take a 'Config' to control how the

430−-- instance is generated.

455+-- | @deriveSerializeWith config-modifier instance-dec@ generates a template

456+-- Haskell splice consisting of a declaration of a 'Serialize' instance.

457+-- @instance-dec@ is a template Haskell declaration splice consisting of a

458+-- standard Haskell instance declaration without the type class methods (e.g.

459+-- @[d|instance Serialize a => Serialize (Maybe a)|]@).

460+--

461+-- The type class methods for the given instance are generated according to the

462+-- supplied @config-modifier@ parameter. See 'SerializeConfig' for default

463+-- configuration settings.

431464--

432465-- Usage:

466+--

433467-- @

434−-- $(deriveSerializeWith

435−-- config

468+-- \$(deriveSerializeWith

469+-- ( inlineSerializeAt (Just NoInline)

470+-- . inlineDeserializeAt (Just NoInline)

471+-- )

436472-- [d|instance Serialize a => Serialize (Maybe a)|])

437473-- @

438−deriveSerializeWith :: Config -> Q [Dec] -> Q [Dec]

439−deriveSerializeWith conf mDecs = do

474+deriveSerializeWith ::

475+ (SerializeConfig -> SerializeConfig) -> Q [Dec] -> Q [Dec]

476+deriveSerializeWith modifier mDecs = do

440477 dec <- mDecs

441478 case dec of

442− [inst@(InstanceD _ _ headTyWC [])] -> do

479+ [InstanceD mo preds headTyWC []] -> do

443480 let headTy = unwrap dec headTyWC

444481 dt <- reifyDataType (getMainTypeName dec headTy)

445482 let cons = dtCons dt

446− deriveSerializeInternal conf headTy cons (next inst)

447− _ -> errorUnsupported (errorMessage dec)

448− where

483+ deriveSerializeInternal

484+ (modifier serializeConfig) headTy cons (next mo preds headTyWC)

485+ _ -> errorMessage dec

486+

487+ where

449488

450− next (InstanceD mo preds headTyWC []) methods =

451− pure [InstanceD mo preds headTyWC methods]

452− next dec _ = errorUnsupported (errorMessage dec)

489+ next mo preds headTyWC methods = pure [InstanceD mo preds headTyWC methods]

453490

454491 errorMessage dec =

455492 error $ unlines

@@ -457,8 +494,8 @@ deriveSerializeWith conf mDecs = do

457494 , ""

458495 , ">> " ++ pprint dec

459496 , ""

460− , "The above is not a valid instance declaration."

461− , "Any haskell instance declaration without a body is valid."

497+ , "The supplied declaration is not a valid instance declaration."

498+ , "Provide a valid Haskell instance declaration without a body."

462499 , ""

463500 , "Examples:"

464501 , "instance Serialize (Proxy a)"

@@ -477,37 +514,17 @@ deriveSerializeWith conf mDecs = do

477514 go (AppT l _) = go l

478515 go _ = errorMessage dec

479516

480−

481−-- | Template haskell helper to create instances of 'Serialize' automatically.

517+-- | Given an 'Serialize' instance declaration splice without the methods (e.g.

518+-- @[d|instance Serialize a => Serialize (Maybe a)|]@), generate an instance

519+-- declaration including all the type class method implementations.

482520--

483−-- Consider the datatype:

484−-- @

485−-- data CustomDataType a b c = ...

486−-- @

521+-- >>> deriveSerialize = deriveSerializeWith id

487522--

488−-- Usage: @$(deriveSerialize ''CustomDataType)@

523+-- Usage:

489524--

490−-- Note: All type variables automatcally get an "Serialize" constraint.

491−-- The derived code will look like the following,

492525-- @

493−-- instance (Serialize a, Serialize b, Serialize c) => Serialize (CustomDataType a b c) where

494−-- ...

526+-- \$(deriveSerialize

527+-- [d|instance Serialize a => Serialize (Maybe a)|])

495528-- @

496−--

497−-- To control which type variables don't get the Serialize constraint, use

498−-- 'deriveSerializeWith'.

499−deriveSerialize :: Name -> Q [Dec]

500−deriveSerialize name = do

501− dt <- reifyDataType name

502− let preds = map (unboxPred . VarT) (dtTvs dt)

503− headTy = appsT (ConT name) (map VarT (dtTvs dt))

504− cons = dtCons dt

505− deriveSerializeInternal defaultConfig headTy cons (next preds headTy)

506−

507−

508− where

509−

510− next preds headTy methods =

511− pure [InstanceD Nothing preds (AppT (ConT ''Serialize) headTy) methods]

512−

513− unboxPred ty = AppT (ConT ''Serialize) ty

529+deriveSerialize :: Q [Dec] -> Q [Dec]

530+deriveSerialize = deriveSerializeWith id

core/src/Streamly/Internal/Data/Serialize/TH/Bottom.hs+143−48

@@ -9,8 +9,17 @@

99-- Portability : GHC

1010--

1111module Streamly.Internal.Data.Serialize.TH.Bottom

12− ( Config(..)

13− , defaultConfig

12+ (

13+ -- ** Config

14+ SerializeConfig(..)

15+ , serializeConfig

16+ , inlineAddSizeTo

17+ , inlineSerializeAt

18+ , inlineDeserializeAt

19+ , encodeConstrNames

20+ , encodeRecordFields

21+

22+ -- ** Other Utilities

1423 , TypeOfType(..)

1524 , typeOfType

1625 , SimpleDataCon(..)

@@ -65,48 +74,134 @@ import Streamly.Internal.Data.Unbox.TH (DataCon(..))

6574-- Config

6675--------------------------------------------------------------------------------

6776

68−-- | Config to control how the 'Serialize' instance is generated.

69−data Config =

70− Config

71− { inlineSize :: Inline

72− -- ^ Inline value for 'size'. Default is Inline.

73− , inlineSerialize :: Inline

74− -- ^ Inline value for 'serialize'. Default is Inline.

75− , inlineDeserialize :: Inline

76− -- ^ Inline value for 'deserialize'. Default is Inline.

77− , constructorTagAsString :: Bool

78− -- ^ __Experimental__

79− --

80− -- If True, encode constructors using the constructor names as Latin-1

81− -- byte sequence.

82− , recordSyntaxWithHeader :: Bool

83− -- ^ __Experimental__

84− --

85− -- If True, encode the keys of the record as a header and then

86− -- serialize the data. Multiple constructors are not supported with

87− -- @recordSyntaxWithHeader@ enabled.

88− --

89− -- __Performance Notes:__

90− --

91− -- There is a constant regression proportional to

92− -- @sum (map length keyList) + length keyList@ where @keyList@ is the

93− -- list of keys of that record as strings.

94− --

95− -- As an example, @keyList@ of,

96− -- @

97− -- data RecordType = RecordType { field0 :: Int, field2 :: Char }

98− -- @

99− -- is @["field0", "field1"]@

77+-- NOTE: 'Nothing' is not eqvivalant to 'Just Inlinable'. Ie. Having no inline

78+-- specific pragma and having an Inlinable pragma are different. Having an

79+-- Inlinable pragma makes GHC put the code in the interface file whereas having

80+-- no inline specific pragma let's GHC decide whether to put the code in

81+-- interface file or not.

82+

83+-- | Configuration to control how the 'Serialize' instance is generated. The

84+-- configuration is opaque and is modified by composing config modifier

85+-- functions, for example:

86+--

87+-- >>> (inlineSerializeAt (Just NoInline)) . (inlineSerializeAt (Just Inlinable))

88+--

89+-- The default configuration settings are:

90+--

91+-- * 'inlineAddSizeTo' Nothing

92+-- * 'inlineSerializeAt' (Just Inline)

93+-- * 'inlineDeserializeAt' (Just Inline)

94+--

95+-- The following experimental options are also available:

96+--

97+-- * 'encodeConstrNames' False

98+-- * 'encodeRecordFields' False

99+--

100+data SerializeConfig =

101+ SerializeConfig

102+ { cfgInlineSize :: Maybe Inline

103+ , cfgInlineSerialize :: Maybe Inline

104+ , cfgInlineDeserialize :: Maybe Inline

105+ , cfgConstructorTagAsString :: Bool

106+ , cfgRecordSyntaxWithHeader :: Bool

100107 }

101108

102−defaultConfig :: Config

103−defaultConfig =

104− Config

105− { inlineSize = Inline

106− , inlineSerialize = Inline

107− , inlineDeserialize = Inline

108− , constructorTagAsString = False

109− , recordSyntaxWithHeader = False

109+-- | How should we inline the 'addSizeTo' function? The default is 'Nothing'

110+-- which means left to the compiler. Forcing inline on @addSizeTo@ function

111+-- actually worsens some benchmarks and improves none.

112+inlineAddSizeTo :: Maybe Inline -> SerializeConfig -> SerializeConfig

113+inlineAddSizeTo v cfg = cfg {cfgInlineSize = v}

114+

115+-- XXX Should we make the default Inlinable instead?

116+

117+-- | How should we inline the 'serialize' function? The default 'Just Inline'.

118+-- However, aggressive inlining can bloat the code and increase in compilation

119+-- times when there are big functions and too many nesting levels so you can

120+-- change it accordingly. A 'Nothing' value leaves the decision to the

121+-- compiler.

122+inlineSerializeAt :: Maybe Inline -> SerializeConfig -> SerializeConfig

123+inlineSerializeAt v cfg = cfg {cfgInlineSerialize = v}

124+

125+-- XXX Should we make the default Inlinable instead?

126+

127+-- | How should we inline the 'deserialize' function? See guidelines in

128+-- 'inlineSerializeAt'.

129+inlineDeserializeAt :: Maybe Inline -> SerializeConfig -> SerializeConfig

130+inlineDeserializeAt v cfg = cfg {cfgInlineDeserialize = v}

131+

132+-- | __Experimental__

133+--

134+-- In sum types, use Latin-1 encoded original constructor names rather than

135+-- binary values to identify constructors. This option is not applicable to

136+-- product types.

137+--

138+-- This option enables the following behavior:

139+--

140+-- * __Reordering__: Order of the fields can be changed without affecting

141+-- serialization.

142+-- * __Addition__: If a field is added in the new version, the old version of

143+-- the data type can still be deserialized by the new version. The new value

144+-- would never occur in the old one.

145+-- * __Deletion__: If a field is deleted in the new version, deserialization

146+-- of the old version will result in an error. TBD: We can possibly designate a

147+-- catch-all case to handle this scenario.

148+--

149+-- Note that if you change a type, change the semantics of a type, or delete a

150+-- field and add a new field with the same name, deserialization of old data

151+-- may result in silent unexpected behavior.

152+--

153+-- This option has to be the same on both encoding and decoding side.

154+--

155+-- The default is 'False'.

156+--

157+encodeConstrNames :: Bool -> SerializeConfig -> SerializeConfig

158+encodeConstrNames v cfg = cfg {cfgConstructorTagAsString = v}

159+

160+-- XXX We can deserialize each field to Either, so if there is a

161+-- deserialization error in any field it can handled independently. Also, a

162+-- unique type/version identifier of the field (based on the versions of the

163+-- packages, full module name space + type identifier) can be serialized along

164+-- with the value for stricter compatibility, semantics checking. Or we can

165+-- store a type hash.

166+

167+-- | __Experimental__

168+--

169+-- In explicit record types, use Latin-1 encoded record field names rather than

170+-- binary values to identify the record fields. Note that this option is not

171+-- applicable to sum types. Also, it does not work on a product type which is

172+-- not a record, because there are no field names to begin with.

173+--

174+-- This option enables the following behavior:

175+--

176+-- * __Reordering__: Order of the fields can be changed without affecting

177+-- serialization.

178+-- * __Addition__: If a 'Maybe' type field is added in the new version, the old

179+-- version of the data type can still be deserialized by the new version, the

180+-- field value in the older version is assumed to be 'Nothing'. If any other

181+-- type of field is added, deserialization of the older version results in an

182+-- error but only when that field is actually accessed in the deserialized

183+-- record.

184+-- * __Deletion__: If a field is deleted in the new version and it is

185+-- encountered in a previously serialized version then the field is discarded.

186+--

187+-- This option has to be the same on both encoding and decoding side.

188+--

189+-- There is a constant performance overhead proportional to the total length of

190+-- the record field names and the number of record fields.

191+--

192+-- The default is 'False'.

193+--

194+encodeRecordFields :: Bool -> SerializeConfig -> SerializeConfig

195+encodeRecordFields v cfg = cfg {cfgRecordSyntaxWithHeader = v}

196+

197+serializeConfig :: SerializeConfig

198+serializeConfig =

199+ SerializeConfig

200+ { cfgInlineSize = Nothing

201+ , cfgInlineSerialize = Just Inline

202+ , cfgInlineDeserialize = Just Inline

203+ , cfgConstructorTagAsString = False

204+ , cfgRecordSyntaxWithHeader = False

110205 }

111206

112207--------------------------------------------------------------------------------

@@ -252,7 +347,7 @@ xorCmp tag off arr =

252347 go8 i = do

253348 let wIntegral = litIntegral i

254349 [|xor (unsafeInlineIO

255− (Unbox.peekByteIndex

350+ (Unbox.peekAt

256351 ($(varE off) + $(litIntegral i))

257352 $(varE arr)))

258353 ($(wIntegral) :: Word8) .|.

@@ -266,7 +361,7 @@ xorCmp tag off arr =

266361 litIntegral

267362 (shiftAdd w8_w16 [tag !! i, tag !! (i + 1)] :: Word16)

268363 [|xor (unsafeInlineIO

269− (Unbox.peekByteIndex

364+ (Unbox.peekAt

270365 ($(varE off) + $(litIntegral i))

271366 $(varE arr)))

272367 ($(wIntegral) :: Word16) .|.

@@ -286,7 +381,7 @@ xorCmp tag off arr =

286381 , tag !! (i + 3)

287382 ] :: Word32)

288383 [|xor (unsafeInlineIO

289− (Unbox.peekByteIndex

384+ (Unbox.peekAt

290385 ($(varE off) + $(litIntegral i))

291386 $(varE arr)))

292387 ($(wIntegral) :: Word32) .|.

@@ -310,7 +405,7 @@ xorCmp tag off arr =

310405 , tag !! (i + 7)

311406 ])

312407 [|xor (unsafeInlineIO

313− (Unbox.peekByteIndex

408+ (Unbox.peekAt

314409 ($(varE off) + $(litIntegral i))

315410 $(varE arr)))

316411 ($(wIntegral) :: Word64) .|.

@@ -336,7 +431,7 @@ serializeW8List off arr w8List = do

336431 makeBind i =

337432 bindS

338433 (varP (makeN (i + 1)))

339− [|$(varE 'serialize)

434+ [|$(varE 'serializeAt)

340435 $(varE (makeN i))

341436 $(varE arr)

342437 ($(litIntegral (w8List !! i)) :: Word8)|]

@@ -361,7 +456,7 @@ errorUnsupported err =

361456 $ unlines

362457 [ "Unsupported"

363458 , "==========="

364− , "There is improper use of the library."

459+ , "This is improper use of the library."

365460 , "This case is unsupported."

366461 , "Please contact the developer if this case is of interest."

367462 , ""

core/src/Streamly/Internal/Data/Serialize/TH/RecHeader.hs+37−37

@@ -9,9 +9,9 @@

99-- Portability : GHC

1010--

1111module Streamly.Internal.Data.Serialize.TH.RecHeader

12− ( mkSerializeExpr

13− , mkDeserializeExpr

14− , mkSizeOfExpr

12+ ( mkRecSerializeExpr

13+ , mkRecDeserializeExpr

14+ , mkRecSizeOfExpr

1515 , conUpdateFuncDec

1616 , mkDeserializeKeysDec

1717 ) where

@@ -27,7 +27,7 @@ import Data.Maybe (fromJust)

2727import Language.Haskell.TH

2828import Streamly.Internal.Data.Serialize.Type (Serialize(..))

2929import Data.Foldable (foldlM)

30−import Streamly.Internal.Data.Unbox (MutableByteArray)

30+import Streamly.Internal.Data.MutByteArray.Type (MutByteArray)

3131import Data.Proxy (Proxy(..))

3232

3333import qualified Streamly.Internal.Data.Unbox as Unbox

@@ -103,36 +103,36 @@ newtype CompactList a =

103103-- the list is 255.

104104instance forall a. Serialize a => Serialize (CompactList a) where

105105

106− -- {-# INLINE size #-}

107− size acc (CompactList xs) =

108− foldl' size (acc + (Unbox.sizeOf (Proxy :: Proxy Word8))) xs

106+ -- {-# INLINE addSizeTo #-}

107+ addSizeTo acc (CompactList xs) =

108+ foldl' addSizeTo (acc + (Unbox.sizeOf (Proxy :: Proxy Word8))) xs

109109

110110 -- Inlining this causes large compilation times for tests

111− {-# INLINABLE deserialize #-}

112− deserialize off arr sz = do

113− (off1, len8) <- deserialize off arr sz :: IO (Int, Word8)

111+ {-# INLINABLE deserializeAt #-}

112+ deserializeAt off arr sz = do

113+ (off1, len8) <- deserializeAt off arr sz :: IO (Int, Word8)

114114 let len = w8_int len8

115115 peekList f o i | i >= 3 = do

116116 -- Unfold the loop three times

117− (o1, x1) <- deserialize o arr sz

118− (o2, x2) <- deserialize o1 arr sz

119− (o3, x3) <- deserialize o2 arr sz

117+ (o1, x1) <- deserializeAt o arr sz

118+ (o2, x2) <- deserializeAt o1 arr sz

119+ (o3, x3) <- deserializeAt o2 arr sz

120120 peekList (f . (\xs -> x1:x2:x3:xs)) o3 (i - 3)

121121 peekList f o 0 = pure (o, f [])

122122 peekList f o i = do

123− (o1, x) <- deserialize o arr sz

123+ (o1, x) <- deserializeAt o arr sz

124124 peekList (f . (x:)) o1 (i - 1)

125125 (nextOff, lst) <- peekList id off1 len

126126 pure (nextOff, CompactList lst)

127127

128128 -- Inlining this causes large compilation times for tests

129− {-# INLINABLE serialize #-}

130− serialize off arr (CompactList val) = do

131− void $ serialize off arr (int_w8 (length val) :: Word8)

129+ {-# INLINABLE serializeAt #-}

130+ serializeAt off arr (CompactList val) = do

131+ void $ serializeAt off arr (int_w8 (length val) :: Word8)

132132 let off1 = off + Unbox.sizeOf (Proxy :: Proxy Word8)

133133 let pokeList o [] = pure o

134134 pokeList o (x:xs) = do

135− o1 <- serialize o arr x

135+ o1 <- serializeAt o arr x

136136 pokeList o1 xs

137137 pokeList off1 val

138138

@@ -154,7 +154,7 @@ isMaybeType _ = False

154154-- We add 4 here because we use 'serializeWithSize' for serializing.

155155exprGetSize :: Q Exp -> (Int, Type) -> Q Exp

156156exprGetSize acc (i, _) =

157− [|size $(acc) $(varE (mkFieldName i)) + 4|]

157+ [|addSizeTo $(acc) $(varE (mkFieldName i)) + 4|]

158158

159159sizeOfHeader :: SimpleDataCon -> Int

160160sizeOfHeader (SimpleDataCon _ fields) =

@@ -171,8 +171,8 @@ sizeOfHeader (SimpleDataCon _ fields) =

171171 sizeForNumFields = 1 -- At max 255 fields in the record constructor

172172 sizeForFieldLen = 1 -- At max 255 letters in the key

173173

174−mkSizeOfExpr :: SimpleDataCon -> Q Exp

175−mkSizeOfExpr con = do

174+mkRecSizeOfExpr :: SimpleDataCon -> Q Exp

175+mkRecSizeOfExpr con = do

176176 n_acc <- newName "acc"

177177 n_x <- newName "x"

178178 (lamE

@@ -225,27 +225,27 @@ headerValue (SimpleDataCon _ fields) =

225225-- its type. We encode the size as 'Word32' hence there is a 4 bytes increase

226226-- in size.

227227{-# INLINE serializeWithSize #-}

228−serializeWithSize :: Serialize a => Int -> MutableByteArray -> a -> IO Int

228+serializeWithSize :: Serialize a => Int -> MutByteArray -> a -> IO Int

229229serializeWithSize off arr val = do

230− off1 <- serialize (off + 4) arr val

231− Unbox.pokeByteIndex off arr (int_w32 (off1 - off - 4) :: Word32)

230+ off1 <- serializeAt (off + 4) arr val

231+ Unbox.pokeAt off arr (int_w32 (off1 - off - 4) :: Word32)

232232 pure off1

233233

234−mkSerializeExpr :: Name -> SimpleDataCon -> Q Exp

235−mkSerializeExpr initialOffset (con@(SimpleDataCon cname fields)) = do

234+mkRecSerializeExpr :: Name -> SimpleDataCon -> Q Exp

235+mkRecSerializeExpr initialOffset (con@(SimpleDataCon cname fields)) = do

236236 afterHLen <- newName "afterHLen"

237237 -- Encoding the header length is required.

238238 -- We first compare the header length encoded and the current header

239239 -- length. Only if the header lengths match, we compare the headers.

240240 [|do $(varP afterHLen) <-

241− serialize

241+ serializeAt

242242 ($(varE initialOffset) + 4)

243243 $(varE _arr)

244244 ($(litIntegral hlen) :: Word32)

245245 $(varP (makeI 0)) <- $(serializeW8List afterHLen _arr hval)

246246 let $(openConstructor cname (length fields)) = $(varE _val)

247247 finalOff <- $(mkSerializeExprFields 'serializeWithSize fields)

248− Unbox.pokeByteIndex

248+ Unbox.pokeAt

249249 $(varE initialOffset)

250250 $(varE _arr)

251251 ((fromIntegral :: Int -> Word32)

@@ -263,8 +263,8 @@ mkSerializeExpr initialOffset (con@(SimpleDataCon cname fields)) = do

263263

264264{-# INLINE deserializeWithSize #-}

265265deserializeWithSize ::

266− Serialize a => Int -> MutableByteArray -> Int -> IO (Int, a)

267−deserializeWithSize off arr endOff = deserialize (off + 4) arr endOff

266+ Serialize a => Int -> MutByteArray -> Int -> IO (Int, a)

267+deserializeWithSize off arr endOff = deserializeAt (off + 4) arr endOff

268268

269269conUpdateFuncDec :: Name -> [Field] -> Q [Dec]

270270conUpdateFuncDec funcName fields = do

@@ -282,7 +282,7 @@ conUpdateFuncDec funcName fields = do

282282 wildP

283283 (normalB

284284 [|do (valOff, valLen :: Word32) <-

285− deserialize

285+ deserializeAt

286286 $(varE curOff)

287287 $(varE arr)

288288 $(varE endOff)

@@ -317,7 +317,7 @@ conUpdateFuncDec funcName fields = do

317317 (litP fnameLit)

318318 (normalB

319319 [|do (valOff, valLen :: Word32) <-

320− deserialize

320+ deserializeAt

321321 $(varE currOff)

322322 $(varE arr)

323323 $(varE endOff)

@@ -345,7 +345,7 @@ mkDeserializeKeysDec funcName updateFunc (SimpleDataCon cname fields) = do

345345 errorUnsupported "The datatype should use record syntax."

346346 method <-

347347 [|do (dataOff, hlist :: CompactList (CompactList Word8)) <-

348− deserialize $(varE hOff) $(varE arr) $(varE endOff)

348+ deserializeAt $(varE hOff) $(varE arr) $(varE endOff)

349349 let keys = wListToString . unCompactList <$> unCompactList hlist

350350 ($(varP kvEncoded), _) <-

351351 foldlM

@@ -383,16 +383,16 @@ mkDeserializeKeysDec funcName updateFunc (SimpleDataCon cname fields) = do

383383 else [|error $(litE (StringL (nameBase k ++ " is not found.")))|]

384384

385385

386−mkDeserializeExpr :: Name -> Name -> Name -> SimpleDataCon -> Q Exp

387−mkDeserializeExpr initialOff endOff deserializeWithKeys con = do

386+mkRecDeserializeExpr :: Name -> Name -> Name -> SimpleDataCon -> Q Exp

387+mkRecDeserializeExpr initialOff endOff deserializeWithKeys con = do

388388 hOff <- newName "hOff"

389389 let sizeForFinalOff = 4 -- Word32

390390 sizeForHeaderLength = 4 -- Word32

391391 sizePreData = sizeForFinalOff + sizeForHeaderLength + hlen

392392 [|do (hlenOff, encLen :: Word32) <-

393− deserialize $(varE initialOff) $(varE _arr) $(varE endOff)

393+ deserializeAt $(varE initialOff) $(varE _arr) $(varE endOff)

394394 ($(varP hOff), hlen1 :: Word32) <-

395− deserialize hlenOff $(varE _arr) $(varE endOff)

395+ deserializeAt hlenOff $(varE _arr) $(varE endOff)

396396 if (hlen1 == $(litIntegral hlen)) && $(xorCmp hval hOff _arr)

397397 then do

398398 let $(varP (makeI 0)) =

core/src/Streamly/Internal/Data/Serialize/Type.hs+140−137

@@ -7,37 +7,27 @@

77--

99module Streamly.Internal.Data.Serialize.Type

10− ( Serialize(..)

11− , encode

12− , pinnedEncode

13− , decode

10+ (

11+ Serialize(..)

1412 ) where

1513

1614--------------------------------------------------------------------------------

1715-- Imports

1816--------------------------------------------------------------------------------

1917

20−#include "assert.hs"

21−

22−#ifdef DEBUG

23−import Control.Exception (assert)

24−#endif

25−

18+import Control.Monad (when)

2619import Data.List (foldl')

2720import Data.Proxy (Proxy (..))

28−import Streamly.Internal.Data.Unbox

29− ( MutableByteArray(..)

30− , PinnedState(..)

31− , Unbox

32− )

21+import Streamly.Internal.Data.Unbox (Unbox)

22+import Streamly.Internal.Data.MutByteArray.Type (MutByteArray(..))

3323import Streamly.Internal.Data.Array.Type (Array(..))

34−import Streamly.Internal.System.IO (unsafeInlineIO)

3524import GHC.Int (Int16(..), Int32(..), Int64(..), Int8(..))

3625import GHC.Word (Word16(..), Word32(..), Word64(..), Word8(..))

3726import GHC.Stable (StablePtr(..))

3827

28+import qualified Streamly.Internal.Data.MutByteArray.Type as MBA

3929import qualified Streamly.Internal.Data.Unbox as Unbox

40−import qualified Streamly.Internal.Data.Array as Array

30+import qualified Streamly.Internal.Data.Array.Type as Array

4131import qualified Streamly.Internal.Data.MutArray as MutArray

4232

4333import GHC.Exts

@@ -59,21 +49,33 @@ import GHC.Exts

5949-- Types

6050--------------------------------------------------------------------------------

6151

62−-- | A type implementing the 'Serialize' interface supplies operations for

63−-- reading and writing the type from and to a mutable byte array (an unboxed

64−-- representation of the type) in memory. The read operation 'deserialize'

65−-- deserializes the boxed type from the mutable byte array. The write operation

66−-- 'serialize' serializes the boxed type to the mutable byte array.

52+-- | The 'Serialize' type class provides operations for serialization and

53+-- deserialization of general Haskell data types to and from their byte stream

54+-- representation.

55+--

56+-- Unlike 'Unbox', 'Serialize' uses variable length encoding, therefore, it can

57+-- serialize recursive and variable length data types like lists, or variable

58+-- length sum types where the length of the value may vary depending on a

59+-- particular constructor. For variable length data types the length is encoded

60+-- along with the data.

6761--

68−-- IMPORTANT: The serialized data's byte ordering is not normalized, which means

69−-- it remains the same as the machine's byte order where the function is

70−-- executed. Consequently, it may not be compatible across machines with

71−-- different byte ordering.

62+-- The 'deserializeAt' operation reads bytes from the mutable byte array and

63+-- builds a Haskell data type from these bytes, the number of bytes it reads

64+-- depends on the type and the encoded value it is reading. 'serializeAt'

65+-- operation converts a Haskell data type to its binary representation which

66+-- must consist of as many bytes as added by the @addSizeTo@ operation for that

67+-- value and then stores these bytes into the mutable byte array. The

68+-- programmer is expected to use the @addSizeTo@ operation and allocate an

69+-- array of sufficient length before calling 'serializeAt'.

7270--

73−-- 'Serialize' contains enough information to serialize and deserialize variable

74−-- length types.

71+-- IMPORTANT: The serialized data's byte ordering remains the same as the host

72+-- machine's byte order. Therefore, it can not be deserialized from host

73+-- machines with a different byte ordering.

7574--

76−-- >>> import Streamly.Internal.Data.Serialize.Type (Serialize(..))

75+-- Instances can be derived via Template Haskell, or written manually.

76+--

77+-- Here is an example, for deriving an instance of this type class using

78+-- template Haskell:

7779--

7880-- >>> :{

7981-- data Object = Object

@@ -82,16 +84,30 @@ import GHC.Exts

8284-- }

8385-- :}

8486--

87+-- @

88+-- import Streamly.Data.MutByteArray (deriveSerialize)

89+-- \$(deriveSerialize [d|instance Serialize Object|])

90+-- @

91+--

92+-- See 'Streamly.Data.MutByteArray.deriveSerialize' and

93+-- 'Streamly.Data.MutByteArray.deriveSerializeWith' for more information on

94+-- deriving using Template Haskell.

95+--

96+-- Here is an example of a manual instance.

97+--

98+-- >>> import Streamly.Data.MutByteArray (Serialize(..))

99+--

85100-- >>> :{

86101-- instance Serialize Object where

87−-- size acc obj = size (size acc (_obj1 obj)) (_obj2 obj)

88−-- deserialize i arr len = do

89−-- (i1, x0) <- deserialize i arr len

90−-- (i2, x1) <- deserialize i1 arr len

102+-- addSizeTo acc obj = addSizeTo (addSizeTo acc (_obj1 obj)) (_obj2 obj)

103+-- deserializeAt i arr len = do

104+-- -- Check the array bounds before reading

105+-- (i1, x0) <- deserializeAt i arr len

106+-- (i2, x1) <- deserializeAt i1 arr len

91107-- pure (i2, Object x0 x1)

92−-- serialize i arr (Object x0 x1) = do

93−-- i1 <- serialize i arr x0

94−-- i2 <- serialize i1 arr x1

108+-- serializeAt i arr (Object x0 x1) = do

109+-- i1 <- serializeAt i arr x0

110+-- i2 <- serializeAt i1 arr x1

95111-- pure i2

96112-- :}

97113--

@@ -102,44 +118,50 @@ class Serialize a where

102118 -- It is of the form @Int -> a -> Int@ because you can have tail-recursive

103119 -- traversal of the structures.

104120

105− -- | Get the size, in bytes, reqired to store the serialized

106− -- representation of the type. Size cannot be zero.

107− size :: Int -> a -> Int

121+ -- | @addSizeTo accum value@ returns @accum@ incremented by the size of the

122+ -- serialized representation of @value@ in bytes. Size cannot be zero. It

123+ -- should be at least 1 byte.

124+ addSizeTo :: Int -> a -> Int

108125

109126 -- We can implement the following functions without returning the `Int`

110127 -- offset but that may require traversing the Haskell structure again to get

111128 -- the size. Therefore, this is a performance optimization.

112129

113− -- | @deserialize offset array arrayLen@ deserializes a value from the

114− -- given byte-index in the array. Returns a tuple of the next byte-index

115− -- and the deserialized value.

116− deserialize :: Int -> MutableByteArray -> Int -> IO (Int, a)

117−

118− -- | Write the serialized representation of the value in the array at the

119− -- given byte-index. Returns the next byte-index. This is an unsafe

120− -- operation, the programmer must ensure that the array has enough space

121− -- available in the array to serialize the value as determined by the

122− -- @size@ operation.

123− serialize :: Int -> MutableByteArray -> a -> IO Int

130+ -- | @deserializeAt byte-offset array arrayLen@ deserializes a value from

131+ -- the given byte-offset in the array. Returns a tuple consisting of the

132+ -- next byte-offset and the deserialized value.

133+ --

134+ -- The arrayLen passed is the entire length of the input buffer. It is to

135+ -- be used to check if we would overflow the input buffer when

136+ -- deserializing.

137+ --

138+ -- Throws an exception if the operation would exceed the supplied arrayLen.

139+ deserializeAt :: Int -> MutByteArray -> Int -> IO (Int, a)

140+

141+ -- | @serializeAt byte-offset array value@ writes the serialized

142+ -- representation of the @value@ in the array at the given byte-offset.

143+ -- Returns the next byte-offset.

144+ --

145+ -- This is an unsafe operation, the programmer must ensure that the array

146+ -- has enough space available to serialize the value as determined by the

147+ -- @addSizeTo@ operation.

148+ serializeAt :: Int -> MutByteArray -> a -> IO Int

124149

125150--------------------------------------------------------------------------------

126151-- Instances

127152--------------------------------------------------------------------------------

128153

129−-- XXX We do not need to pass the end of data offset, we can just check the

130−-- mutable array end here to avoid a crash, and when we return from deserialize

131−-- we can check if the offset returned is beyond the bound or not.

132−--

154+-- _size is the length from array start to the last accessed byte.

133155#ifdef DEBUG

134156{-# INLINE checkBounds #-}

135−checkBounds :: String -> Int -> MutableByteArray -> IO ()

136−checkBounds _label _off _arr = do

137− sz <- sizeOfMutableByteArray _arr

138− if (_off > sz)

157+checkBounds :: String -> Int -> MutByteArray -> IO ()

158+checkBounds _label _size _arr = do

159+ sz <- MBA.sizeOfMutableByteArray _arr

160+ if (_size > sz)

139161 then error

140162 $ _label

141163 ++ ": accessing array at offset = "

142− ++ show (_off - 1)

164+ ++ show (_size - 1)

143165 ++ " max valid offset = " ++ show (sz - 1)

144166 else return ()

145167#endif

@@ -162,39 +184,39 @@ checkBounds _label _off _arr = do

162184-- single array for deserializing. But that can also be done by framing the

163185-- serialized value with a size header.

164186--

165−{-# INLINE deserializeUnsafe #-}

166−deserializeUnsafe :: forall a. Unbox a => Int -> MutableByteArray -> Int -> IO (Int, a)

167−deserializeUnsafe off arr sz =

187+{-# INLINE deserializeChecked #-}

188+deserializeChecked :: forall a. Unbox a => Int -> MutByteArray -> Int -> IO (Int, a)

189+deserializeChecked off arr sz =

168190 let next = off + Unbox.sizeOf (Proxy :: Proxy a)

169191 in do

170192 -- Keep likely path in the straight branch.

171193 if (next <= sz)

172− then Unbox.peekByteIndex off arr >>= \val -> pure (next, val)

194+ then Unbox.peekAt off arr >>= \val -> pure (next, val)

173195 else error

174− $ "deserialize: accessing array at offset = "

196+ $ "deserializeAt: accessing array at offset = "

175197 ++ show (next - 1)

176198 ++ " max valid offset = " ++ show (sz - 1)

177199

178200{-# INLINE serializeUnsafe #-}

179−serializeUnsafe :: forall a. Unbox a => Int -> MutableByteArray -> a -> IO Int

201+serializeUnsafe :: forall a. Unbox a => Int -> MutByteArray -> a -> IO Int

180202serializeUnsafe off arr val =

181203 let next = off + Unbox.sizeOf (Proxy :: Proxy a)

182204 in do

183205#ifdef DEBUG

184− checkBounds "serialize" next arr

206+ checkBounds "serializeAt" next arr

185207#endif

186− Unbox.pokeByteIndex off arr val

208+ Unbox.pokeAt off arr val

187209 pure next

188210

189211#define DERIVE_SERIALIZE_FROM_UNBOX(_type) \

190212instance Serialize _type where \

191−; {-# INLINE size #-} \

192−; size acc _ = acc + Unbox.sizeOf (Proxy :: Proxy _type) \

193−; {-# INLINE deserialize #-} \

194−; deserialize off arr end = deserializeUnsafe off arr end :: IO (Int, _type) \

195−; {-# INLINE serialize #-} \

196−; serialize = \

197− serializeUnsafe :: Int -> MutableByteArray -> _type -> IO Int

213+; {-# INLINE addSizeTo #-} \

214+; addSizeTo acc _ = acc + Unbox.sizeOf (Proxy :: Proxy _type) \

215+; {-# INLINE deserializeAt #-} \

216+; deserializeAt off arr end = deserializeChecked off arr end :: IO (Int, _type) \

217+; {-# INLINE serializeAt #-} \

218+; serializeAt = \

219+ serializeUnsafe :: Int -> MutByteArray -> _type -> IO Int

198220

199221DERIVE_SERIALIZE_FROM_UNBOX(())

200222DERIVE_SERIALIZE_FROM_UNBOX(Bool)

@@ -217,98 +239,79 @@ DERIVE_SERIALIZE_FROM_UNBOX((FunPtr a))

217239

218240instance forall a. Serialize a => Serialize [a] where

219241

220− -- {-# INLINE size #-}

221− size acc xs = foldl' size (acc + (Unbox.sizeOf (Proxy :: Proxy Int))) xs

242+ -- {-# INLINE addSizeTo #-}

243+ addSizeTo acc xs =

244+ foldl' addSizeTo (acc + (Unbox.sizeOf (Proxy :: Proxy Int))) xs

222245

223246 -- Inlining this causes large compilation times for tests

224− {-# INLINABLE deserialize #-}

225− deserialize off arr sz = do

226− (off1, len64) <- deserialize off arr sz :: IO (Int, Int64)

247+ {-# INLINABLE deserializeAt #-}

248+ deserializeAt off arr sz = do

249+ (off1, len64) <- deserializeAt off arr sz :: IO (Int, Int64)

227250 let len = (fromIntegral :: Int64 -> Int) len64

228251 peekList f o i | i >= 3 = do

229252 -- Unfold the loop three times

230− (o1, x1) <- deserialize o arr sz

231− (o2, x2) <- deserialize o1 arr sz

232− (o3, x3) <- deserialize o2 arr sz

253+ (o1, x1) <- deserializeAt o arr sz

254+ (o2, x2) <- deserializeAt o1 arr sz

255+ (o3, x3) <- deserializeAt o2 arr sz

233256 peekList (f . (\xs -> x1:x2:x3:xs)) o3 (i - 3)

234257 peekList f o 0 = pure (o, f [])

235258 peekList f o i = do

236− (o1, x) <- deserialize o arr sz

259+ (o1, x) <- deserializeAt o arr sz

237260 peekList (f . (x:)) o1 (i - 1)

238261 peekList id off1 len

239262

240263 -- Inlining this causes large compilation times for tests

241− {-# INLINABLE serialize #-}

242− serialize off arr val = do

264+ {-# INLINABLE serializeAt #-}

265+ serializeAt off arr val = do

243266 let off1 = off + Unbox.sizeOf (Proxy :: Proxy Int64)

244267 let pokeList acc o [] =

245− Unbox.pokeByteIndex off arr (acc :: Int64) >> pure o

268+ Unbox.pokeAt off arr (acc :: Int64) >> pure o

246269 pokeList acc o (x:xs) = do

247− o1 <- serialize o arr x

270+ o1 <- serializeAt o arr x

248271 pokeList (acc + 1) o1 xs

249272 pokeList 0 off1 val

250273

251−instance Serialize (Array a) where

252− {-# INLINE size #-}

253− size i (Array {..}) = i + (arrEnd - arrStart) + 8

274+instance

275+#ifdef DEVBUILD

276+ Unbox a =>

277+#endif

278+ Serialize (Array a) where

279+ {-# INLINE addSizeTo #-}

280+ addSizeTo i (Array {..}) = i + (arrEnd - arrStart) + 8

254281

255− {-# INLINE deserialize #-}

256− deserialize off arr end = do

257− (off1, byteLen) <- deserialize off arr end :: IO (Int, Int)

282+ {-# INLINE deserializeAt #-}

283+ deserializeAt off arr len = do

284+ (off1, byteLen) <- deserializeAt off arr len :: IO (Int, Int)

258285 let off2 = off1 + byteLen

286+ when (off2 > len) $

287+ error

288+ $ "deserializeAt: accessing array at offset = "

289+ ++ show (off2 - 1)

290+ ++ " max valid offset = " ++ show (len - 1)

291+ -- XXX Use MutByteArray.cloneSliceUnsafe

259292 let slice = MutArray.MutArray arr off1 off2 off2

260293 newArr <- MutArray.clone slice

261294 pure (off2, Array.unsafeFreeze newArr)

262295

263− {-# INLINE serialize #-}

264− serialize off arr (Array {..}) = do

296+ {-# INLINE serializeAt #-}

297+ serializeAt off arr (Array {..}) = do

265298 let arrLen = arrEnd - arrStart

266− off1 <- serialize off arr arrLen

267− Unbox.putSliceUnsafe arrContents arrStart arr off1 arrLen

299+ off1 <- serializeAt off arr arrLen

300+ MBA.putSliceUnsafe arrContents arrStart arr off1 arrLen

268301 pure (off1 + arrLen)

269302

270303instance (Serialize a, Serialize b) => Serialize (a, b) where

271304

272− {-# INLINE size #-}

273− size acc (a, b) = size (size acc a) b

305+ {-# INLINE addSizeTo #-}

306+ addSizeTo acc (a, b) = addSizeTo (addSizeTo acc a) b

274307

275− {-# INLINE serialize #-}

276− serialize off arr (a, b) = do

277− off1 <- serialize off arr a

278− serialize off1 arr b

308+ {-# INLINE serializeAt #-}

309+ serializeAt off arr (a, b) = do

310+ off1 <- serializeAt off arr a

311+ serializeAt off1 arr b

279312

280− {-# INLINE deserialize #-}

281− deserialize off arr end = do

282− (off1, a) <- deserialize off arr end

283− (off2, b) <- deserialize off1 arr end

313+ {-# INLINE deserializeAt #-}

314+ deserializeAt off arr end = do

315+ (off1, a) <- deserializeAt off arr end

316+ (off2, b) <- deserializeAt off1 arr end

284317 pure (off2, (a, b))

285−

286−--------------------------------------------------------------------------------

287−-- High level functions

288−--------------------------------------------------------------------------------

289−

290−{-# INLINE encodeAs #-}

291−encodeAs :: forall a. Serialize a => PinnedState -> a -> Array Word8

292−encodeAs ps a =

293− unsafeInlineIO $ do

294− let len = size 0 a

295− mbarr <- Unbox.newBytesAs ps len

296− off <- serialize 0 mbarr a

297− assertM(len == off)

298− pure $ Array mbarr 0 off

299−

300−{-# INLINE encode #-}

301−encode :: Serialize a => a -> Array Word8

302−encode = encodeAs Unpinned

303−

304−{-# INLINE pinnedEncode #-}

305−pinnedEncode :: Serialize a => a -> Array Word8

306−pinnedEncode = encodeAs Pinned

307−

308−{-# INLINE decode #-}

309−decode :: Serialize a => Array Word8 -> a

310−decode arr@(Array {..}) = unsafeInlineIO $ do

311− let lenArr = Array.length arr

312− (off, val) <- deserialize arrStart arrContents (arrStart + lenArr)

313− assertM(off == arrStart + lenArr)

314− pure val

core/src/Streamly/Internal/Data/Stream/Container.hs+56−48

@@ -11,16 +11,17 @@

1111

1212module Streamly.Internal.Data.Stream.Container

1313 (

14− nub

14+ -- * Deduplication

15+ ordNub

1516

16− -- * Joins for unconstrained types

17− , joinLeftGeneric

18− , joinOuterGeneric

17+ -- * Joins

18+ , leftJoin

19+ , outerJoin

1920

20− -- * Joins with Ord constraint

21− , joinInner

22− , joinLeft

23− , joinOuter

21+ -- * Ord Joins

22+ , innerOrdJoin

23+ , leftOrdJoin

24+ , outerOrdJoin

2425 )

2526where

2627

@@ -47,12 +48,15 @@ import qualified Streamly.Internal.Data.Stream.Transformer as Stream

4748

4849#include "DocTestDataStream.hs"

4950

50−-- | The memory used is proportional to the number of unique elements in the

51−-- stream. If we want to limit the memory we can just use "take" to limit the

52−-- uniq elements in the stream.

53−{-# INLINE_NORMAL nub #-}

54−nub :: (Monad m, Ord a) => Stream m a -> Stream m a

55−nub (Stream step1 state1) = Stream step (Set.empty, state1)

51+-- | @nub@ specialized to 'Ord' types for better performance. Returns a

52+-- subsequence of the stream removing any duplicate elements.

53+--

54+-- The memory used is proportional to the number of unique elements in the

55+-- stream. One way to limit the memory is to use @take@ on the resulting

56+-- stream to limit the unique elements in the stream.

57+{-# INLINE_NORMAL ordNub #-}

58+ordNub :: (Monad m, Ord a) => Stream m a -> Stream m a

59+ordNub (Stream step1 state1) = Stream step (Set.empty, state1)

5660

5761 where

5862

@@ -78,8 +82,8 @@ toMap =

7882--

7983-- XXX An IntMap may be faster when the keys are Int.

8084-- XXX Use hashmap instead of map?

81−--

82−-- | Like 'joinInner' but uses a 'Map' for efficiency.

85+

86+-- | 'innerJoin' specialized to 'Ord' types for better performance.

8387--

8488-- If the input streams have duplicate keys, the behavior is undefined.

8589--

@@ -90,10 +94,10 @@ toMap =

9094-- Time: O(m + n)

9195--

9296-- /Pre-release/

93−{-# INLINE joinInner #-}

94−joinInner :: (Monad m, Ord k) =>

97+{-# INLINE innerOrdJoin #-}

98+innerOrdJoin :: (Monad m, Ord k) =>

9599 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, b)

96−joinInner s1 s2 =

100+innerOrdJoin s1 s2 =

97101 Stream.concatEffect $ do

98102 km <- toMap s2

99103 pure $ Stream.mapMaybe (joinAB km) s1

@@ -106,32 +110,33 @@ joinInner s1 s2 =

106110 Nothing -> Nothing

107111

108112-- XXX We can do this concurrently.

113+-- XXX Check performance of StreamD vs StreamK

109114-- XXX If the second stream is sorted and passed as an Array or a seek capable

110115-- stream then we could use binary search if we have an Ord instance or

111116-- Ordering returning function. The time complexity would then become (m x log

112117-- n).

113118

114−-- XXX Check performance of StreamD vs StreamK

115−

116−-- | Like 'joinInner' but emit @(a, Just b)@, and additionally, for those @a@'s

117−-- that are not equal to any @b@ emit @(a, Nothing)@.

119+-- | Like 'innerJoin' but emits @(a, Just b)@ whenever a and b are equal, for

120+-- those @a@'s that are not equal to any @b@ emits @(a, Nothing)@.

118121--

119−-- The second stream is evaluated multiple times. If the stream is a

120−-- consume-once stream then the caller should cache it in an 'Data.Array.Array'

121−-- before calling this function. Caching may also improve performance if the

122−-- stream is expensive to evaluate.

122+-- This is a generalization of 'innerJoin' to include all elements from the

123+-- left stream and not just those which have an equal in the right stream. This

124+-- is not a commutative operation, the order of the stream arguments matters.

123125--

124−-- >>> joinRightGeneric eq = flip (Stream.joinLeftGeneric eq)

126+-- All the caveats mentioned in 'innerJoin' apply here as well. Right join is

127+-- not provided because it is just a flipped left join:

128+--

129+-- >>> rightJoin eq = flip (Stream.leftJoin eq)

125130--

126131-- Space: O(n) assuming the second stream is cached in memory.

127132--

128133-- Time: O(m x n)

129134--

130135-- /Unimplemented/

131−{-# INLINE joinLeftGeneric #-}

132−joinLeftGeneric :: Monad m =>

136+{-# INLINE leftJoin #-}

137+leftJoin :: Monad m =>

133138 (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)

134−joinLeftGeneric eq s1 s2 = Stream.evalStateT (return False) $ unCross $ do

139+leftJoin eq s1 s2 = Stream.evalStateT (return False) $ unCross $ do

135140 a <- mkCross (Stream.liftInner s1)

136141 -- XXX should we use StreamD monad here?

137142 -- XXX Is there a better way to perform some action at the end of a loop

@@ -152,19 +157,17 @@ joinLeftGeneric eq s1 s2 = Stream.evalStateT (return False) $ unCross $ do

152157 else mkCross Stream.nil

153158 Nothing -> return (a, Nothing)

154159

155−-- XXX rename to joinLeftOrd?

156−

157−-- | A more efficient 'joinLeft' using a hashmap for efficiency.

160+-- | 'leftJoin' specialized to 'Ord' types for better performance.

158161--

159162-- Space: O(n)

160163--

161164-- Time: O(m + n)

162165--

163166-- /Pre-release/

164−{-# INLINE joinLeft #-}

165−joinLeft :: (Ord k, Monad m) =>

167+{-# INLINE leftOrdJoin #-}

168+leftOrdJoin :: (Ord k, Monad m) =>

166169 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, a, Maybe b)

167−joinLeft s1 s2 =

170+leftOrdJoin s1 s2 =

168171 Stream.concatEffect $ do

169172 km <- toMap s2

170173 return $ fmap (joinAB km) s1

@@ -177,14 +180,19 @@ joinLeft s1 s2 =

177180 Nothing -> (k, a, Nothing)

178181

179182-- XXX We can do this concurrently.

183+-- XXX Check performance of StreamD vs StreamK cross operation.

180184

181−-- XXX Check performance of StreamD vs StreamK

182−

183−-- | Like 'joinLeft' but emits a @(Just a, Just b)@. Like 'joinLeft', for those

185+-- | Like 'leftJoin' but emits a @(Just a, Just b)@. Like 'leftJoin', for those

184186-- @a@'s that are not equal to any @b@ emit @(Just a, Nothing)@, but

185187-- additionally, for those @b@'s that are not equal to any @a@ emit @(Nothing,

186188-- Just b)@.

187189--

190+-- This is a generalization of left join to include all the elements from the

191+-- right stream as well, in other words it is a combination of left and right

192+-- joins. This is a commutative operation. The order of stream arguments can be

193+-- changed without affecting results, except for the ordering of elements in

194+-- the resulting tuple.

195+--

188196-- For space efficiency use the smaller stream as the second stream.

189197--

190198-- Space: O(n)

@@ -192,15 +200,15 @@ joinLeft s1 s2 =

192200-- Time: O(m x n)

193201--

194202-- /Pre-release/

195−{-# INLINE joinOuterGeneric #-}

196−joinOuterGeneric :: MonadIO m =>

203+{-# INLINE outerJoin #-}

204+outerJoin :: MonadIO m =>

197205 (a -> b -> Bool)

198206 -> Stream m a

199207 -> Stream m b

200208 -> Stream m (Maybe a, Maybe b)

201−joinOuterGeneric eq s1 s =

209+outerJoin eq s1 s2 =

202210 Stream.concatEffect $ do

203− inputArr <- Array.fromStream s

211+ inputArr <- Array.fromStream s2

204212 let len = Array.length inputArr

205213 foundArr <-

206214 Stream.fold

@@ -254,18 +262,18 @@ joinOuterGeneric eq s1 s =

254262-- a flag. At the end go through @Stream m b@ and find those that are not in that

255263-- hash to return (Nothing, b).

256264

257−-- | Like 'joinOuter' but uses a 'Map' for efficiency.

265+-- | 'outerJoin' specialized to 'Ord' types for better performance.

258266--

259267-- Space: O(m + n)

260268--

261269-- Time: O(m + n)

262270--

263271-- /Pre-release/

264−{-# INLINE joinOuter #-}

265−joinOuter ::

272+{-# INLINE outerOrdJoin #-}

273+outerOrdJoin ::

266274 (Ord k, MonadIO m) =>

267275 Stream m (k, a) -> Stream m (k, b) -> Stream m (k, Maybe a, Maybe b)

268−joinOuter s1 s2 =

276+outerOrdJoin s1 s2 =

269277 Stream.concatEffect $ do

270278 km1 <- kvFold s1

271279 km2 <- kvFold s2

core/src/Streamly/Internal/Data/Stream/Eliminate.hs+33−25

@@ -13,36 +13,24 @@

1313--

1414module Streamly.Internal.Data.Stream.Eliminate

1515 (

16− -- * Running a 'Fold'

17− fold

18−

19− -- -- * Running a 'Parser'

20− , parse

16+ -- * Running a Parser

17+ parse

2118 , parseD

2219 , parseBreak

2320 , parseBreakD

2421

25− -- * Stream Deconstruction

22+ -- * Deconstruction

2623 , uncons

2724

2825 -- * Right Folds

29− , foldrM

30− , foldr

31− , foldrMx

3226 , foldr1

3327

34− -- * Left Folds

35− , foldlM'

36− , foldl'

37− , foldlMx'

38− , foldlx'

39−

4028 -- * Specific Fold Functions

41− , drain

4229 , mapM_ -- Map and Fold

4330 , null

4431 , head

4532 , headElse

33+ , init

4634 , tail

4735 , last

4836 , elem

@@ -60,16 +48,9 @@ module Streamly.Internal.Data.Stream.Eliminate

6048 , the

6149

6250 -- * To containers

63− , toList

6451 , toListRev

6552

6653 -- * Multi-Stream Folds

67− -- ** Comparisons

68− -- | These should probably be expressed using zipping operations.

69− , eqBy

70− , cmpBy

71−

72− -- ** Substreams

7354 -- | These should probably be expressed using parsers.

7455 , isPrefixOf

7556 , isInfixOf

@@ -90,7 +71,7 @@ import Foreign.Storable (Storable)

9071import GHC.Exts (SpecConstrAnnotation(..))

9172import GHC.Types (SPEC(..))

9273import Streamly.Internal.Data.Parser (ParseError(..))

93−import Streamly.Internal.Data.SVar.Type (defState)

74+import Streamly.Internal.Data.SVar.Type (adaptState, defState)

9475import Streamly.Internal.Data.Unbox (Unbox)

9576

9677import Streamly.Internal.Data.Maybe.Strict (Maybe'(..))

@@ -105,7 +86,7 @@ import qualified Streamly.Internal.Data.Stream.Transform as StreamD

10586

10687import Prelude hiding

10788 ( Foldable(..), all, any, head, last, lookup, mapM, mapM_

108− , notElem, splitAt, tail, (!!))

89+ , notElem, splitAt, init, tail, (!!))

10990import Data.Foldable (length)

11091import Streamly.Internal.Data.Stream.Type

11192

@@ -391,9 +372,35 @@ head = foldrM (\x _ -> return (Just x)) (return Nothing)

391372headElse :: Monad m => a -> Stream m a -> m a

392373headElse a = foldrM (\x _ -> return x) (return a)

393374

375+{-# INLINE_NORMAL init #-}

376+init :: Monad m => Stream m a -> m (Maybe (Stream m a))

377+init stream = do

378+ r <- uncons stream

379+ case r of

380+ Nothing -> return Nothing

381+ Just (h, Stream step1 state1) ->

382+ return $ Just $ Stream step (h, state1)

383+

384+ where

385+

386+ step gst (a, s1) = do

387+ res <- step1 (adaptState gst) s1

388+ return $

389+ case res of

390+ Yield x s -> Yield a (x, s)

391+ Skip s -> Skip (a, s)

392+ Stop -> Stop

393+

394+-- | Same as:

395+--

396+-- >>> tail = fmap (fmap snd) . Stream.uncons

397+--

394398-- Does not fuse, has the same performance as the StreamK version.

399+--

395400{-# INLINE_NORMAL tail #-}

396401tail :: Monad m => Stream m a -> m (Maybe (Stream m a))

402+tail = fmap (fmap snd) . uncons

403+{-

397404tail (UnStream step state) = go SPEC state

398405 where

399406 go !_ st = do

@@ -402,6 +409,7 @@ tail (UnStream step state) = go SPEC state

402409 Yield _ s -> return (Just $ Stream step s)

403410 Skip s -> go SPEC s

404411 Stop -> return Nothing

412+-}

405413

406414-- XXX will it fuse? need custom impl?

407415{-# INLINE_NORMAL last #-}

core/src/Streamly/Internal/Data/Stream/Exception.hs+9−6

@@ -9,17 +9,20 @@

1010module Streamly.Internal.Data.Stream.Exception

1111 (

12− gbracket_

13− , gbracket

14− , before

15− , afterUnsafe

12+ -- * Resources

13+ before

1614 , afterIO

15+ , afterUnsafe

16+ , finallyIO

17+ , finallyUnsafe

18+ , gbracket_

19+ , gbracket

1720 , bracketUnsafe

1821 , bracketIO3

1922 , bracketIO

23+

24+ -- * Exceptions

2025 , onException

21− , finallyUnsafe

22− , finallyIO

2326 , ghandle

2427 , handle

2528 )

core/src/Streamly/Internal/Data/Stream/Generate.hs+2−12

@@ -16,20 +16,13 @@ module Streamly.Internal.Data.Stream.Generate

1616 (

1717 -- * Primitives

1818 nil

19− , nilM

2019 , cons

21− , consM

22−

23− -- * From 'Unfold'

24− , unfold

2520

2621 -- * Unfolding

2722 , unfoldr

2823 , unfoldrM

2924

3025 -- * From Values

31− , fromPure

32− , fromEffect

3326 , repeat

3427 , repeatM

3528 , replicate

@@ -95,7 +88,6 @@ module Streamly.Internal.Data.Stream.Generate

9588 -- * From Containers

9689 -- | Transform an input structure into a stream.

9790

98− , fromList

9991 , fromListM

10092 , fromFoldable

10193 , fromFoldableM

@@ -104,10 +96,6 @@ module Streamly.Internal.Data.Stream.Generate

10496 , fromPtr

10597 , fromPtrN

10698 , fromByteStr#

107−

108− -- * Conversions

109− , fromStreamK

110− , toStreamK

11199 )

112100where

113101

@@ -154,6 +142,8 @@ import Streamly.Internal.Data.Stream.Type

154142nil :: Applicative m => Stream m a

155143nil = Stream (\_ _ -> pure Stop) ()

156144

145+infixr 5 `cons`

146+

157147-- XXX implement in terms of consM?

158148-- cons x = consM (return x)

159149-- From an implementation perspective, StreamK.cons translates into a

core/src/Streamly/Internal/Data/Stream/MutChunked.hsdeleted−331

@@ -1,331 +0,0 @@

1−-- |

2−-- Module : Streamly.Internal.Data.Stream.MutChunked

4−-- License : BSD3-3-Clause

5−-- Maintainer : streamly@composewell.com

6−-- Stability : experimental

7−-- Portability : GHC

8−--

9−-- Combinators to efficiently manipulate streams of mutable arrays.

10−--

11−module Streamly.Internal.Data.Stream.MutChunked

12− (

13− -- * Generation

14− chunksOf

15−

16− -- * Compaction

17− , packArraysChunksOf

18− , SpliceState (..)

19− , lpackArraysChunksOf

20− , compact

21− , compactLE

22− , compactEQ

23− , compactGE

24− )

25−where

26−

27−#include "inline.hs"

28−#include "ArrayMacros.h"

29−

30−import Control.Monad.IO.Class (MonadIO(..))

31−import Control.Monad (when)

32−import Data.Bifunctor (first)

33−import Data.Proxy (Proxy(..))

34−import Streamly.Internal.Data.Unbox (Unbox, sizeOf)

35−import Streamly.Internal.Data.MutArray.Type (MutArray(..))

36−import Streamly.Internal.Data.Fold.Type (Fold(..))

37−import Streamly.Internal.Data.Parser (ParseError)

38−import Streamly.Internal.Data.Stream.Type (Stream)

39−import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))

40−

41−import qualified Streamly.Internal.Data.MutArray.Type as MArray

42−import qualified Streamly.Internal.Data.Fold.Type as FL

43−import qualified Streamly.Internal.Data.Parser as ParserD

44−import qualified Streamly.Internal.Data.Stream as D

45−

46−-- | @chunksOf n stream@ groups the elements in the input stream into arrays of

47−-- @n@ elements each.

48−--

49−-- Same as the following but may be more efficient:

50−--

51−-- > chunksOf n = Stream.foldMany (MArray.writeN n)

52−--

53−-- /Pre-release/

54−{-# INLINE chunksOf #-}

55−chunksOf :: (MonadIO m, Unbox a)

56− => Int -> Stream m a -> Stream m (MutArray a)

57−chunksOf = MArray.chunksOf

58−

59−-------------------------------------------------------------------------------

60−-- Compact

61−-------------------------------------------------------------------------------

62−

63−data SpliceState s arr

64− = SpliceInitial s

65− | SpliceBuffering s arr

66− | SpliceYielding arr (SpliceState s arr)

67− | SpliceFinish

68−

69−-- XXX This can be removed once compactLEFold/compactLE are implemented.

70−--

71−-- | This mutates the first array (if it has space) to append values from the

72−-- second one. This would work for immutable arrays as well because an

73−-- immutable array never has space so a new array is allocated instead of

74−-- mutating it.

75−--

76−-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

77−-- maximum specified size. Note that if a single array is bigger than the

78−-- specified size we do not split it to fit. When we coalesce multiple arrays

79−-- if the size would exceed the specified size we do not coalesce therefore the

80−-- actual array size may be less than the specified chunk size.

81−--

82−-- @since 0.7.0

83−{-# INLINE_NORMAL packArraysChunksOf #-}

84−packArraysChunksOf :: (MonadIO m, Unbox a)

85− => Int -> D.Stream m (MutArray a) -> D.Stream m (MutArray a)

86−packArraysChunksOf n (D.Stream step state) =

87− D.Stream step' (SpliceInitial state)

88−

89− where

90−

91− {-# INLINE_LATE step' #-}

92− step' gst (SpliceInitial st) = do

93− when (n <= 0) $

94− -- XXX we can pass the module string from the higher level API

95− error $ "Streamly.Internal.Data.MutArray.Type.packArraysChunksOf: the size of "

96− ++ "arrays [" ++ show n ++ "] must be a natural number"

97− r <- step gst st

98− case r of

99− D.Yield arr s -> return $

100− let len = MArray.byteLength arr

101− in if len >= n

102− then D.Skip (SpliceYielding arr (SpliceInitial s))

103− else D.Skip (SpliceBuffering s arr)

104− D.Skip s -> return $ D.Skip (SpliceInitial s)

105− D.Stop -> return D.Stop

106−

107− step' gst (SpliceBuffering st buf) = do

108− r <- step gst st

109− case r of

110− D.Yield arr s -> do

111− let len = MArray.byteLength buf + MArray.byteLength arr

112− if len > n

113− then return $

114− D.Skip (SpliceYielding buf (SpliceBuffering s arr))

115− else do

116− buf' <- if MArray.byteCapacity buf < n

117− then liftIO $ MArray.realloc n buf

118− else return buf

119− buf'' <- MArray.splice buf' arr

120− return $ D.Skip (SpliceBuffering s buf'')

121− D.Skip s -> return $ D.Skip (SpliceBuffering s buf)

122− D.Stop -> return $ D.Skip (SpliceYielding buf SpliceFinish)

123−

124− step' _ SpliceFinish = return D.Stop

125−

126− step' _ (SpliceYielding arr next) = return $ D.Yield arr next

127−

128−-- XXX Remove this once compactLEFold is implemented

129−-- lpackArraysChunksOf = Fold.many compactLEFold

130−--

131−{-# INLINE_NORMAL lpackArraysChunksOf #-}

132−lpackArraysChunksOf :: (MonadIO m, Unbox a)

133− => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()

134−lpackArraysChunksOf n (Fold step1 initial1 _ final1) =

135− Fold step initial extract final

136−

137− where

138−

139− initial = do

140− when (n <= 0) $

141− -- XXX we can pass the module string from the higher level API

142− error $ "Streamly.Internal.Data.MutArray.Type.packArraysChunksOf: the size of "

143− ++ "arrays [" ++ show n ++ "] must be a natural number"

144−

145− r <- initial1

146− return $ first (Tuple' Nothing) r

147−

148− step (Tuple' Nothing r1) arr =

149− let len = MArray.byteLength arr

150− in if len >= n

151− then do

152− r <- step1 r1 arr

153− case r of

154− FL.Done _ -> return $ FL.Done ()

155− FL.Partial s -> do

156− _ <- final1 s

157− res <- initial1

158− return $ first (Tuple' Nothing) res

159− else return $ FL.Partial $ Tuple' (Just arr) r1

160−

161− step (Tuple' (Just buf) r1) arr = do

162− let len = MArray.byteLength buf + MArray.byteLength arr

163− buf' <- if MArray.byteCapacity buf < len

164− then liftIO $ MArray.realloc (max n len) buf

165− else return buf

166− buf'' <- MArray.splice buf' arr

167−

168− -- XXX this is common in both the equations of step

169− if len >= n

170− then do

171− r <- step1 r1 buf''

172− case r of

173− FL.Done _ -> return $ FL.Done ()

174− FL.Partial s -> do

175− _ <- final1 s

176− res <- initial1

177− return $ first (Tuple' Nothing) res

178− else return $ FL.Partial $ Tuple' (Just buf'') r1

179−

180− -- XXX Several folds do extract >=> final, therefore, we need to make final

181− -- return "m b" rather than using extract post it if we want extract to be

182− -- partial.

183− --

184− -- extract forces the pending buffer to be sent to the fold which is not

185− -- what we want.

186− extract _ = error "lpackArraysChunksOf: not designed for scanning"

187−

188− final (Tuple' Nothing r1) = final1 r1

189− final (Tuple' (Just buf) r1) = do

190− r <- step1 r1 buf

191− case r of

192− FL.Partial rr -> final1 rr

193− FL.Done _ -> return ()

194−

195−-- XXX Same as compactLE, to be removed once that is implemented.

196−--

197−-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

198−-- maximum specified size in bytes.

199−--

200−-- /Internal/

201−{-# INLINE compact #-}

202−compact :: (MonadIO m, Unbox a)

203− => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

204−compact = packArraysChunksOf

205−

206−-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

207−-- maximum specified size. Note that if a single array is bigger than the

208−-- specified size we do not split it to fit. When we coalesce multiple arrays

209−-- if the size would exceed the specified size we do not coalesce therefore the

210−-- actual array size may be less than the specified chunk size.

211−--

212−-- /Internal/

213−{-# INLINE_NORMAL compactLEParserD #-}

214−compactLEParserD ::

215− forall m a. (MonadIO m, Unbox a)

216− => Int -> ParserD.Parser (MutArray a) m (MutArray a)

217−compactLEParserD n = ParserD.Parser step initial extract

218−

219− where

220−

221− nBytes = n * SIZE_OF(a)

222−

223− initial =

224− return

225− $ if n <= 0

226− then error

227− $ functionPath

228− ++ ": the size of arrays ["

229− ++ show n ++ "] must be a natural number"

230− else ParserD.IPartial Nothing

231−

232− step Nothing arr =

233− return

234− $ let len = MArray.byteLength arr

235− in if len >= nBytes

236− then ParserD.Done 0 arr

237− else ParserD.Partial 0 (Just arr)

238− step (Just buf) arr =

239− let len = MArray.byteLength buf + MArray.byteLength arr

240− in if len > nBytes

241− then return $ ParserD.Done 1 buf

242− else do

243− buf1 <-

244− if MArray.byteCapacity buf < nBytes

245− then liftIO $ MArray.realloc nBytes buf

246− else return buf

247− buf2 <- MArray.splice buf1 arr

248− return $ ParserD.Partial 0 (Just buf2)

249−

250− extract Nothing = return $ ParserD.Done 0 MArray.nil

251− extract (Just buf) = return $ ParserD.Done 0 buf

252−

253− functionPath =

254− "Streamly.Internal.Data.Stream.MutChunked.compactLEParserD"

255−

256−-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

257−-- minimum specified size. Note that if all the arrays in the stream together

258−-- are smaller than the specified size the resulting array will be smaller than

259−-- the specified size. When we coalesce multiple arrays if the size would exceed

260−-- the specified size we stop coalescing further.

261−--

262−-- /Internal/

263−{-# INLINE_NORMAL compactGEFold #-}

264−compactGEFold ::

265− forall m a. (MonadIO m, Unbox a)

266− => Int -> FL.Fold m (MutArray a) (MutArray a)

267−compactGEFold n = Fold step initial extract extract

268−

269− where

270−

271− nBytes = n * SIZE_OF(a)

272−

273− initial =

274− return

275− $ if n < 0

276− then error

277− $ functionPath

278− ++ ": the size of arrays ["

279− ++ show n ++ "] must be a natural number"

280− else FL.Partial Nothing

281−

282− step Nothing arr =

283− return

284− $ let len = MArray.byteLength arr

285− in if len >= nBytes

286− then FL.Done arr

287− else FL.Partial (Just arr)

288− step (Just buf) arr = do

289− let len = MArray.byteLength buf + MArray.byteLength arr

290− buf1 <-

291− if MArray.byteCapacity buf < len

292− then liftIO $ MArray.realloc (max len nBytes) buf

293− else return buf

294− buf2 <- MArray.splice buf1 arr

295− if len >= n

296− then return $ FL.Done buf2

297− else return $ FL.Partial (Just buf2)

298−

299− extract Nothing = return MArray.nil

300− extract (Just buf) = return buf

301−

302− functionPath =

303− "Streamly.Internal.Data.Stream.MutChunked.compactGEFold"

304−

305−-- | Coalesce adjacent arrays in incoming stream to form bigger arrays of a

306−-- maximum specified size in bytes.

307−--

308−-- /Internal/

309−compactLE :: (MonadIO m, Unbox a) =>

310− Int -> Stream m (MutArray a) -> Stream m (Either ParseError (MutArray a))

311−compactLE n = D.parseManyD (compactLEParserD n)

312−

313−-- | Like 'compactLE' but generates arrays of exactly equal to the size

314−-- specified except for the last array in the stream which could be shorter.

315−--

316−-- /Unimplemented/

317−{-# INLINE compactEQ #-}

318−compactEQ :: -- (MonadIO m, Unbox a) =>

319− Int -> Stream m (MutArray a) -> Stream m (MutArray a)

320−compactEQ _n _xs = undefined

321− -- IsStream.fromStreamD $ D.foldMany (compactEQFold n) (IsStream.toStreamD xs)

322−

323−-- | Like 'compactLE' but generates arrays of size greater than or equal to the

324−-- specified except for the last array in the stream which could be shorter.

325−--

326−-- /Internal/

327−{-# INLINE compactGE #-}

328−compactGE ::

329− (MonadIO m, Unbox a)

330− => Int -> Stream m (MutArray a) -> Stream m (MutArray a)

331−compactGE n = D.foldMany (compactGEFold n)

core/src/Streamly/Internal/Data/Stream/Nesting.hs+19−146

@@ -55,11 +55,6 @@ module Streamly.Internal.Data.Stream.Nesting

5555 -- yields an element. A special case of unfoldRoundRobin.

5656 , roundRobin -- interleaveFair?/ParallelFair

5757

58− -- *** Zipping

59− -- | Zip corresponding elements of two streams.

60− , zipWith

61− , zipWithM

62−

6358 -- *** Merging

6459 -- | Interleave elements from two streams based on a condition.

6560 , mergeBy

@@ -76,17 +71,10 @@ module Streamly.Internal.Data.Stream.Nesting

7671 -- unfoldMany: Unfold m a b -> t m a -> t m b

7772 -- @

7873

79− -- *** ConcatMap

80− -- | Generate streams by mapping a stream generator on each element of an

81− -- input stream, append the resulting streams and flatten.

82− , concatMap

83− , concatMapM

84−

8574 -- *** ConcatUnfold

8675 -- | Generate streams by using an unfold on each element of an input

8776 -- stream, append the resulting streams and flatten. A special case of

8877 -- gintercalate.

89− , unfoldMany

9078 , ConcatUnfoldInterleaveState (..)

9179 , unfoldInterleave

9280 , unfoldRoundRobin

@@ -118,11 +106,8 @@ module Streamly.Internal.Data.Stream.Nesting

118106

119107 -- ** Folding

120108 -- | Apply folds on a stream.

121− , foldMany

122− , refoldMany

123109 , foldSequence

124110 , foldIterateM

125− , refoldIterateM

126111

127112 -- ** Parsing

128113 -- | Parsing is opposite to flattening. 'parseMany' is dual to concatMap or

@@ -139,7 +124,6 @@ module Streamly.Internal.Data.Stream.Nesting

139124

140125 -- ** Grouping

141126 -- | Group segments of a stream and fold. Special case of parsing.

142− , groupsOf

143127 , groupsBy

144128 , groupsWhile

145129 , groupsRollingBy

@@ -147,9 +131,8 @@ module Streamly.Internal.Data.Stream.Nesting

147131 -- ** Splitting

148132 -- | A special case of parsing.

149133 , wordsBy

150− , splitOnSeq

151− , splitOnSuffixSeq

152− , sliceOnSuffix

134+ , splitOnSeq -- XXX splitOnSeg

135+ , splitOnSuffixSeq -- XXX splitOnSegSuffix, splitOnTrailer

153136

154137 -- XXX Implement these as folds or parsers instead.

155138 , splitOnSuffixSeqAny

@@ -158,9 +141,8 @@ module Streamly.Internal.Data.Stream.Nesting

158141

159142 -- * Transform (Nested Containers)

160143 -- | Opposite to compact in ArrayStream

161− , splitInnerBy

162− , splitInnerBySuffix

163− , intersectBySorted

144+ , splitInnerBy -- XXX innerSplitOn

145+ , splitInnerBySuffix -- XXX innerSplitOnSuffix

164146

165147 -- * Reduce By Streams

166148 , dropPrefix

@@ -182,12 +164,9 @@ import Fusion.Plugin.Types (Fuse(..))

182164import GHC.Types (SPEC(..))

183165

184166import Streamly.Internal.Data.Array.Type (Array(..))

185−import Streamly.Internal.Data.Fold.Step (Step(..))

186167import Streamly.Internal.Data.Fold.Type (Fold(..))

187168import Streamly.Internal.Data.Parser (ParseError(..))

188−import Streamly.Internal.Data.Refold.Type (Refold(..))

189169import Streamly.Internal.Data.SVar.Type (adaptState)

190−import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))

191170import Streamly.Internal.Data.Unbox (Unbox, sizeOf)

192171import Streamly.Internal.Data.Unfold.Type (Unfold(..))

193172

@@ -613,49 +592,6 @@ mergeFstBy :: -- Monad m =>

613592mergeFstBy _f _m1 _m2 = undefined

614593 -- fromStreamK $ D.mergeFstBy f (toStreamD m1) (toStreamD m2)

615594

616−-------------------------------------------------------------------------------

617−-- Intersection of sorted streams

618−-------------------------------------------------------------------------------

619−

620−-- Assuming the streams are sorted in ascending order

621−{-# INLINE_NORMAL intersectBySorted #-}

622−intersectBySorted :: Monad m

623− => (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a

624−intersectBySorted cmp (Stream stepa ta) (Stream stepb tb) =

625− Stream step

626− ( ta -- left stream state

627− , tb -- right stream state

628− , Nothing -- left value

629− , Nothing -- right value

630− )

631−

632− where

633−

634− {-# INLINE_LATE step #-}

635− -- step 1, fetch the first value

636− step gst (sa, sb, Nothing, b) = do

637− r <- stepa gst sa

638− return $ case r of

639− Yield a sa' -> Skip (sa', sb, Just a, b) -- step 2/3

640− Skip sa' -> Skip (sa', sb, Nothing, b)

641− Stop -> Stop

642−

643− -- step 2, fetch the second value

644− step gst (sa, sb, a@(Just _), Nothing) = do

645− r <- stepb gst sb

646− return $ case r of

647− Yield b sb' -> Skip (sa, sb', a, Just b) -- step 3

648− Skip sb' -> Skip (sa, sb', a, Nothing)

649− Stop -> Stop

650−

651− -- step 3, compare the two values

652− step _ (sa, sb, Just a, Just b) = do

653− let res = cmp a b

654− return $ case res of

655− GT -> Skip (sa, sb, Just a, Nothing) -- step 2

656− LT -> Skip (sa, sb, Nothing, Just b) -- step 1

657− EQ -> Yield a (sa, sb, Nothing, Just b) -- step 1

658−

659595------------------------------------------------------------------------------

660596-- Combine N Streams - unfoldMany

661597------------------------------------------------------------------------------

@@ -1294,77 +1230,6 @@ foldIterateM func seed0 (Stream step state) =

12941230 stepOuter _ (FIterYield a next) = return $ Yield a next

12951231 stepOuter _ FIterStop = return Stop

12961232

1297−{-# ANN type CIterState Fuse #-}

1298−data CIterState s f fs b

1299− = CIterInit s f

1300− | CIterConsume s fs

1301− | CIterYield b (CIterState s f fs b)

1302− | CIterStop

1303−

1304−-- | Like 'foldIterateM' but using the 'Refold' type instead. This could be

1305−-- much more efficient due to stream fusion.

1306−--

1307−-- /Internal/

1308−{-# INLINE_NORMAL refoldIterateM #-}

1309−refoldIterateM ::

1310− Monad m => Refold m b a b -> m b -> Stream m a -> Stream m b

1311−refoldIterateM (Refold fstep finject fextract) initial (Stream step state) =

1312− Stream stepOuter (CIterInit state initial)

1313−

1314− where

1315−

1316− {-# INLINE iterStep #-}

1317− iterStep st action = do

1318− res <- action

1319− return

1320− $ Skip

1321− $ case res of

1322− FL.Partial fs -> CIterConsume st fs

1323− FL.Done fb -> CIterYield fb $ CIterInit st (return fb)

1324−

1325− {-# INLINE_LATE stepOuter #-}

1326− stepOuter _ (CIterInit st action) = do

1327− iterStep st (action >>= finject)

1328− stepOuter gst (CIterConsume st fs) = do

1329− r <- step (adaptState gst) st

1330− case r of

1331− Yield x s -> iterStep s (fstep fs x)

1332− Skip s -> return $ Skip $ CIterConsume s fs

1333− Stop -> do

1334− b <- fextract fs

1335− return $ Skip $ CIterYield b CIterStop

1336− stepOuter _ (CIterYield a next) = return $ Yield a next

1337− stepOuter _ CIterStop = return Stop

1338−

1339−-- "n" elements at the end are dropped by the fold.

1340−{-# INLINE sliceBy #-}

1341−sliceBy :: Monad m => Fold m a Int -> Int -> Refold m (Int, Int) a (Int, Int)

1342−sliceBy (Fold step1 initial1 extract1 _final) n = Refold step inject extract

1343−

1344− where

1345−

1346− inject (i, len) = do

1347− r <- initial1

1348− return $ case r of

1349− Partial s -> Partial $ Tuple' (i + len + n) s

1350− Done l -> Done (i, l)

1351−

1352− step (Tuple' i s) x = do

1353− r <- step1 s x

1354− return $ case r of

1355− Partial s1 -> Partial $ Tuple' i s1

1356− Done len -> Done (i, len)

1357−

1358− extract (Tuple' i s) = (i,) <$> extract1 s

1359−

1360−{-# INLINE sliceOnSuffix #-}

1361−sliceOnSuffix :: Monad m => (a -> Bool) -> Stream m a -> Stream m (Int, Int)

1362−sliceOnSuffix predicate =

1363− -- Scan the stream with the given refold

1364− refoldIterateM

1365− (sliceBy (FL.takeEndBy_ predicate FL.length) 1)

1366− (return (-1, 0))

1367−

13681233------------------------------------------------------------------------------

13691234-- Parsing

13701235------------------------------------------------------------------------------

@@ -3065,12 +2930,13 @@ splitInnerBy splitter joiner (Stream step1 state1) =

30652930-- | Performs infix separator style splitting.

30662931{-# INLINE_NORMAL splitInnerBySuffix #-}

30672932splitInnerBySuffix

3068− :: (Monad m, Eq (f a), Monoid (f a))

3069− => (f a -> m (f a, Maybe (f a))) -- splitter

2933+ :: Monad m

2934+ => (f a -> Bool) -- isEmpty?

2935+ -> (f a -> m (f a, Maybe (f a))) -- splitter

30702936 -> (f a -> f a -> m (f a)) -- joiner

30712937 -> Stream m (f a)

30722938 -> Stream m (f a)

3073−splitInnerBySuffix splitter joiner (Stream step1 state1) =

2939+splitInnerBySuffix isEmpty splitter joiner (Stream step1 state1) =

30742940 Stream step (SplitInitial state1)

30752941

30762942 where

@@ -3097,10 +2963,11 @@ splitInnerBySuffix splitter joiner (Stream step1 state1) =

30972963 Nothing -> Skip (SplitBuffering s buf')

30982964 Just x2 -> Skip (SplitYielding buf' (SplitSplitting s x2))

30992965 Skip s -> return $ Skip (SplitBuffering s buf)

3100− Stop -> return $

3101− if buf == mempty

3102− then Stop

3103− else Skip (SplitYielding buf SplitFinishing)

2966+ Stop ->

2967+ return $

2968+ if isEmpty buf

2969+ then Stop

2970+ else Skip (SplitYielding buf SplitFinishing)

31042971

31052972 step _ (SplitSplitting st buf) = do

31062973 (x1, mx2) <- splitter buf

@@ -3119,6 +2986,8 @@ splitInnerBySuffix splitter joiner (Stream step1 state1) =

31192986--

31202987-- Space: @O(1)@

31212988--

2989+-- See also stripPrefix.

2990+--

31222991-- /Unimplemented/

31232992{-# INLINE dropPrefix #-}

31242993dropPrefix ::

@@ -3131,6 +3000,8 @@ dropPrefix = error "Not implemented yet!"

31313000--

31323001-- Space: @O(n)@ where n is the length of the infix.

31333002--

3003+-- See also stripInfix.

3004+--

31343005-- /Unimplemented/

31353006{-# INLINE dropInfix #-}

31363007dropInfix ::

@@ -3143,6 +3014,8 @@ dropInfix = error "Not implemented yet!"

31433014--

31443015-- Space: @O(n)@ where n is the length of the suffix.

31453016--

3017+-- See also stripSuffix.

3018+--

31463019-- /Unimplemented/

31473020{-# INLINE dropSuffix #-}

31483021dropSuffix ::

core/src/Streamly/Internal/Data/Stream/Top.hs+243−116

@@ -8,38 +8,73 @@

88-- Portability : GHC

99--

1010-- Top level module that can depend on all other lower level Stream modules.

11+--

12+-- Design notes:

13+--

14+-- The order of arguments in the join operations should ideally be opposite. It

15+-- should be such that the infinite stream is the last one. The transformation

16+-- should be on the last argument, so if you curry the functions with all other

17+-- arguments we get a @Stream -> Stream@ function. The first stream argument

18+-- may be considered as a config or modifier for the operation.

19+--

20+-- Benefit of changing the order is that we get a more intuitive Stream ->

21+-- Stream transformation after currying all other arguments. The inner loop

22+-- streams become arguments for the transformation, more like local modifiers

23+-- for the global outer stream as the last argument. Thus we can continue using

24+-- transformations on the outer stream in a composed pipeline. Otherwise we can

25+-- use flip to flip the order.

26+--

27+-- The fact that the inner stream can be used in the loop multiple times also

28+-- tells that this is not the real effectful stream, it is more like a pure

29+-- stream or an array. In fact we may consider using an Identity streams as

30+-- inner streams in which case these functions will not look nice.

31+--

32+-- Downsides:

33+--

34+-- * Maybe less intuitive to think about, because we usually think the first

35+-- stream as the outer loop and second as the inner.

36+-- * Zip and merge operations will continue using the opposite order.

37+-- * Need to change the order of cross, crossWith operations as well

38+-- * It will be inconsistent with Data.List. The functions cannot be used as

39+-- intuitive operators.

40+--

41+-- The choice is similar to concatMap vs bind. concatMap is pipeline

42+-- composition friendly but bind is user intuition friendly. Another option is

43+-- to have other functions with a different argument order e.g. flippedCross

44+-- instead of cross.

45+--

46+-- If we change the order we have to make sure that we have a consistent

47+-- convention for set-like and the cross join operations.

1148

1249module Streamly.Internal.Data.Stream.Top

1350 (

14− -- * Transformation

15− -- ** Sampling

51+ -- * Sampling

1652 -- | Value agnostic filtering.

1753 strideFromThen

1854

19− -- * Nesting

20− -- ** Set like operations

21− -- | These are not exactly set operations because streams are not

22− -- necessarily sets, they may have duplicated elements. These operations

23− -- are generic i.e. they work on streams of unconstrained types, therefore,

24− -- they have quadratic performance characterstics. For better performance

25− -- using Set structures see the Streamly.Internal.Data.Stream.Container

26− -- module.

27− , filterInStreamGenericBy

28− , deleteInStreamGenericBy

29− , unionWithStreamGenericBy

30−

31− -- ** Set like operations on sorted streams

32− , filterInStreamAscBy

33− , deleteInStreamAscBy

34− , unionWithStreamAscBy

35−

36− -- ** Join operations

37− , joinInnerGeneric

38−

39− -- * Joins on sorted stream

40− , joinInnerAscBy

41− , joinLeftAscBy

42− , joinOuterAscBy

55+ -- * Straight Joins

56+ -- | These are set-like operations but not exactly set operations because

57+ -- streams are not necessarily sets, they may have duplicated elements.

58+ -- These operations are generic i.e. they work on streams of unconstrained

59+ -- types, therefore, they have quadratic performance characterstics. For

60+ -- better performance using Set or Map structures see the

61+ -- Streamly.Internal.Data.Stream.Container module.

62+ , intersectBy

63+ , deleteFirstsBy

64+ , unionBy

65+

66+ -- Set like operations on sorted streams

67+ , sortedIntersectBy

68+ , sortedDeleteFirstsBy

69+ , sortedUnionBy

70+

71+ -- * Cross Joins

72+ , innerJoin

73+

74+ -- Joins on sorted stream

75+ , innerSortedJoin

76+ , leftSortedJoin

77+ , outerSortedJoin

4378 )

4479where

4580

@@ -48,8 +83,7 @@ where

4883import Control.Monad.IO.Class (MonadIO(..))

4984import Data.IORef (newIORef, readIORef, modifyIORef')

5085import Streamly.Internal.Data.Fold.Type (Fold)

51−import Streamly.Internal.Data.Stream.Common ()

52−import Streamly.Internal.Data.Stream.Type (Stream, cross)

86+import Streamly.Internal.Data.Stream.Type (Stream(..), Step(..), cross)

5387

5488import qualified Data.List as List

5589import qualified Streamly.Internal.Data.Fold as Fold

@@ -111,29 +145,40 @@ strideFromThen offset stride =

111145-- binary search if we have an Ord instance or Ordering returning function. The

112146-- time complexity would then become (m x log n).

113147

114−-- | Like 'cross' but emits only those tuples where @a == b@ using the

115−-- supplied equality predicate.

148+-- | Like 'cross' but emits only those tuples where @a == b@ using the supplied

149+-- equality predicate. This is essentially a @cross intersection@ of two

150+-- streams.

116151--

117152-- Definition:

118153--

119−-- >>> joinInnerGeneric eq s1 s2 = Stream.filter (\(a, b) -> a `eq` b) $ Stream.cross s1 s2

154+-- >>> innerJoin eq s1 s2 = Stream.filter (\(a, b) -> a `eq` b) $ Stream.cross s1 s2

155+--

156+-- The second (inner) stream must be finite. Moreover, it must be either pure

157+-- or capable of multiple evaluations. If not then the caller should cache it

158+-- in an 'Data.Array.Array', if the type does not have an 'Unbox' instance then

159+-- use the Generic 'Data.Array.Generic.Array'. Convert the array to stream

160+-- before calling this function. Caching may also improve performance if the

161+-- stream is expensive to evaluate.

120162--

121−-- You should almost always prefer @joinInnerOrd@ over 'joinInnerGeneric' if

122−-- possible. @joinInnerOrd@ is an order of magnitude faster but may take more

123−-- space for caching the second stream.

163+-- If you care about performance this function should be your last choice among

164+-- all inner joins. 'Streamly.Internal.Data.Unfold.innerJoin' is a much faster

165+-- fused alternative. 'innerSortedJoin' is a faster alternative when streams

166+-- are sorted. 'innerOrdJoin' is an order of magnitude faster alternative when

167+-- the type has an 'Ord' instance.

124168--

125−-- See 'Streamly.Internal.Data.Unfold.joinInnerGeneric' for a much faster fused

126−-- alternative.

169+-- Note: Conceptually, this is a commutative operation. Result includes all the

170+-- elements from the left and the right stream. The order of streams can be

171+-- changed without affecting results, except for the ordering within the tuple.

127172--

128173-- Time: O(m x n)

129174--

130175-- /Pre-release/

131−{-# INLINE joinInnerGeneric #-}

132−joinInnerGeneric :: Monad m =>

176+{-# INLINE innerJoin #-}

177+innerJoin :: Monad m =>

133178 (a -> b -> Bool) -> Stream m a -> Stream m b -> Stream m (a, b)

134−joinInnerGeneric eq s1 s2 = Stream.filter (\(a, b) -> a `eq` b) $ cross s1 s2

179+innerJoin eq s1 s2 = Stream.filter (\(a, b) -> a `eq` b) $ cross s1 s2

135180{-

136−joinInnerGeneric eq s1 s2 = do

181+innerJoin eq s1 s2 = do

137182 -- ConcatMap works faster than bind

138183 Stream.concatMap (\a ->

139184 Stream.concatMap (\b ->

@@ -144,44 +189,44 @@ joinInnerGeneric eq s1 s2 = do

144189 ) s1

145190-}

146191

147−-- | A more efficient 'joinInner' for sorted streams.

192+-- | A more efficient 'innerJoin' for sorted streams.

148193--

149194-- Space: O(1)

150195--

151196-- Time: O(m + n)

152197--

153198-- /Unimplemented/

154−{-# INLINE joinInnerAscBy #-}

155−joinInnerAscBy ::

199+{-# INLINE innerSortedJoin #-}

200+innerSortedJoin ::

156201 (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, b)

157−joinInnerAscBy = undefined

202+innerSortedJoin = undefined

158203

159−-- | A more efficient 'joinLeft' for sorted streams.

204+-- | A more efficient 'leftJoin' for sorted streams.

160205--

161206-- Space: O(1)

162207--

163208-- Time: O(m + n)

164209--

165210-- /Unimplemented/

166−{-# INLINE joinLeftAscBy #-}

167−joinLeftAscBy :: -- Monad m =>

211+{-# INLINE leftSortedJoin #-}

212+leftSortedJoin :: -- Monad m =>

168213 (a -> b -> Ordering) -> Stream m a -> Stream m b -> Stream m (a, Maybe b)

169−joinLeftAscBy _eq _s1 _s2 = undefined

214+leftSortedJoin _eq _s1 _s2 = undefined

170215

171−-- | A more efficient 'joinOuter' for sorted streams.

216+-- | A more efficient 'outerJoin' for sorted streams.

172217--

173218-- Space: O(1)

174219--

175220-- Time: O(m + n)

176221--

177222-- /Unimplemented/

178−{-# INLINE joinOuterAscBy #-}

179−joinOuterAscBy :: -- Monad m =>

223+{-# INLINE outerSortedJoin #-}

224+outerSortedJoin :: -- Monad m =>

180225 (a -> b -> Ordering)

181226 -> Stream m a

182227 -> Stream m b

183228 -> Stream m (Maybe a, Maybe b)

184−joinOuterAscBy _eq _s1 _s2 = undefined

229+outerSortedJoin _eq _s1 _s2 = undefined

185230

186231------------------------------------------------------------------------------

187232-- Set operations (special joins)

@@ -191,8 +236,8 @@ joinOuterAscBy _eq _s1 _s2 = undefined

191236-- the best would be to use an Array with linear search. If the second stream

192237-- is sorted we can also use a binary search, using Ord constraint.

193238

194−-- | Keep only those elements in the second stream that are present in the

195−-- first stream too. The first stream is folded to a container using the

239+-- | Keep only those elements in the first stream that are present in the

240+-- second stream too. The second stream is folded to a container using the

196241-- supplied fold and then the elements in the container are looked up using the

197242-- supplied lookup function.

198243--

@@ -207,23 +252,30 @@ filterStreamWith :: Monad m =>

207252filterStreamWith fld member s1 s2 =

208253 Stream.concatEffect

209254 $ do

210− xs <- Stream.fold fld s1

211− return $ Stream.filter (`member` xs) s2

255+ xs <- Stream.fold fld s2

256+ return $ Stream.filter (`member` xs) s1

212257

213−-- | 'filterInStreamGenericBy' retains only those elements in the second stream that

214−-- are present in the first stream.

215−--

216−-- >>> Stream.fold Fold.toList $ Stream.filterInStreamGenericBy (==) (Stream.fromList [1,2,2,4]) (Stream.fromList [2,1,1,3])

217−-- [2,1,1]

218−--

219−-- >>> Stream.fold Fold.toList $ Stream.filterInStreamGenericBy (==) (Stream.fromList [2,1,1,3]) (Stream.fromList [1,2,2,4])

220−-- [1,2,2]

258+-- XXX instead of folding the second stream to a list we could use it directly.

259+-- If the user wants they can generate the stream from an array and also call

260+-- uniq or nub on it. We can provide a convenience Stream -> Stream to cache

261+-- a finite stream in an array and serve it from the cache. The user can decide

262+-- what is best based on the context. They can also choose to use a boxed or

263+-- unboxed array for caching. To force caching we can make the second stream

264+-- monad type Identity. But that may be less flexible. One option is to use

265+-- cachedIntersectBy etc for automatic caching.

266+

267+-- | 'intersectBy' returns a subsequence of the first stream which intersects

268+-- with the second stream. Note that this is not a commutative operation unlike

269+-- a set intersection, because of duplicate elements in the stream the order of

270+-- the streams matters. This is similar to 'Data.List.intersectBy'. Note that

271+-- intersectBy is a special case of 'innerJoin'.

221272--

222−-- Similar to the list intersectBy operation but with the stream argument order

223−-- flipped.

273+-- >>> f s1 s2 = Stream.fold Fold.toList $ Stream.intersectBy (==) (Stream.fromList s1) (Stream.fromList s2)

274+-- >>> f [1,3,4,4,5]) [2,3,4,5,5]

275+-- [3,4,4,5]

224276--

225−-- The first stream must be finite and must not block. Second stream is

226−-- processed only after the first stream is fully realized.

277+-- First stream can be infinite, the second stream must be finite and must be

278+-- capable of multiple evaluations.

227279--

228280-- Space: O(n) where @n@ is the number of elements in the second stream.

229281--

@@ -231,46 +283,87 @@ filterStreamWith fld member s1 s2 =

231283-- @n@ is the number of elements in the second stream.

232284--

233285-- /Pre-release/

234−{-# INLINE filterInStreamGenericBy #-}

235−filterInStreamGenericBy :: Monad m =>

286+{-# INLINE intersectBy #-}

287+intersectBy :: Monad m =>

236288 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a

237−filterInStreamGenericBy eq =

289+intersectBy eq =

238290 -- XXX Use an (unboxed) array instead.

239291 filterStreamWith

240292 (Fold.scanMaybe (Fold.uniqBy eq) Fold.toListRev)

241293 (List.any . eq)

242294

243−-- | Like 'filterInStreamGenericBy' but assumes that the input streams are sorted in

295+-------------------------------------------------------------------------------

296+-- Intersection of sorted streams

297+-------------------------------------------------------------------------------

298+

299+-- XXX The sort order is not important as long both the streams have the same

300+-- sort order. We need to move only in one direction in each stream.

301+-- XXX Fix the argument order to use the same behavior as intersectBy.

302+

303+-- | Like 'intersectBy' but assumes that the input streams are sorted in

244304-- ascending order. To use it on streams sorted in descending order pass an

245305-- inverted comparison function returning GT for less than and LT for greater

246306-- than.

247307--

308+-- Both streams can be infinite.

309+--

248310-- Space: O(1)

249311--

250312-- Time: O(m+n)

251313--

252314-- /Pre-release/

253−{-# INLINE filterInStreamAscBy #-}

254−filterInStreamAscBy :: Monad m =>

315+{-# INLINE_NORMAL sortedIntersectBy #-}

316+sortedIntersectBy :: Monad m =>

255317 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a

256−filterInStreamAscBy eq s1 s2 = Stream.intersectBySorted eq s2 s1

318+sortedIntersectBy cmp (Stream stepa ta) (Stream stepb tb) =

319+ Stream step

320+ ( ta -- left stream state

321+ , tb -- right stream state

322+ , Nothing -- left value

323+ , Nothing -- right value

324+ )

257325

258−-- | Delete all elements of the first stream from the seconds stream. If an

259−-- element occurs multiple times in the first stream as many occurrences of it

260−-- are deleted from the second stream.

261−--

262−-- >>> Stream.fold Fold.toList $ Stream.deleteInStreamGenericBy (==) (Stream.fromList [1,2,3]) (Stream.fromList [1,2,2])

263−-- [2]

326+ where

327+

328+ {-# INLINE_LATE step #-}

329+ -- step 1, fetch the first value

330+ step gst (sa, sb, Nothing, b) = do

331+ r <- stepa gst sa

332+ return $ case r of

333+ Yield a sa' -> Skip (sa', sb, Just a, b) -- step 2/3

334+ Skip sa' -> Skip (sa', sb, Nothing, b)

335+ Stop -> Stop

336+

337+ -- step 2, fetch the second value

338+ step gst (sa, sb, a@(Just _), Nothing) = do

339+ r <- stepb gst sb

340+ return $ case r of

341+ Yield b sb' -> Skip (sa, sb', a, Just b) -- step 3

342+ Skip sb' -> Skip (sa, sb', a, Nothing)

343+ Stop -> Stop

344+

345+ -- step 3, compare the two values

346+ step _ (sa, sb, Just a, Just b) = do

347+ let res = cmp a b

348+ return $ case res of

349+ GT -> Skip (sa, sb, Just a, Nothing) -- step 2

350+ LT -> Skip (sa, sb, Nothing, Just b) -- step 1

351+ EQ -> Yield a (sa, sb, Nothing, Just b) -- step 1

352+

353+-- | Returns a subsequence of the first stream, deleting first occurrences of

354+-- those elements that are present in the second stream. Note that this is not

355+-- a commutative operation. This is similar to the 'Data.List.deleteFirstsBy'.

264356--

265−-- The following laws hold:

357+-- >>> f xs ys = Stream.fold Fold.toList $ Stream.deleteFirstsBy (==) (Stream.fromList xs) (Stream.fromList ys)

358+-- >>> f [1,2,2,3,3,5] [1,2,2,3,4]

359+-- [2,3,5]

266360--

267−-- > deleteInStreamGenericBy (==) s1 (s1 `append` s2) === s2

268−-- > deleteInStreamGenericBy (==) s1 (s1 `interleave` s2) === s2

361+-- The following holds:

269362--

270−-- Same as the list 'Data.List.//' operation but with argument order flipped.

363+-- > deleteFirstsBy (==) (Stream.nub s2 `append` s1) s2 === s1

364+-- > deleteFirstsBy (==) (Stream.nub s2 `interleave` s1) s2 === s1

271365--

272−-- The first stream must be finite and must not block. Second stream is

273−-- processed only after the first stream is fully realized.

366+-- First stream can be infinite, second stream must be finite.

274367--

275368-- Space: O(m) where @m@ is the number of elements in the first stream.

276369--

@@ -278,58 +371,83 @@ filterInStreamAscBy eq s1 s2 = Stream.intersectBySorted eq s2 s1

278371-- @n@ is the number of elements in the second stream.

279372--

280373-- /Pre-release/

281−{-# INLINE deleteInStreamGenericBy #-}

282−deleteInStreamGenericBy :: Monad m =>

374+{-# INLINE deleteFirstsBy #-}

375+deleteFirstsBy :: Monad m =>

283376 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a

284−deleteInStreamGenericBy eq s1 s2 =

285− Stream.concatEffect

286− $ do

287− -- This may work well if s1 is small

288− -- If s1 is big we can go through s1, deleting elements from s2 and

289− -- not emitting an element if it was successfully deleted from s2.

290− -- we will need a deleteBy that can return whether the element was

291− -- deleted or not.

292− xs <- Stream.fold Fold.toList s2

293− let f = Fold.foldl' (flip (List.deleteBy eq)) xs

294− fmap Stream.fromList $ Stream.fold f s1

377+deleteFirstsBy eq s2 s1 =

378+ -- XXX s2 can be a sorted mutable array and we can use binary

379+ -- search to find. Mark the element deleted, count the deletions

380+ -- and reconsolidate the array when a min number of elements is

381+ -- deleted.

295382

296−-- | A more efficient 'deleteInStreamGenericBy' for streams sorted in ascending order.

383+ -- XXX Use StreamK or list as second argument instead of Stream to avoid

384+ -- concatEffect?

385+ Stream.concatEffect $ do

386+ xs <- Stream.toList s1

387+ -- It reverses the list but that is fine.

388+ let del x =

389+ List.foldl' (\(ys,res) y ->

390+ if x `eq` y

391+ then (ys, True)

392+ else (x:ys, res)) ([], False)

393+ g (ys,_) x =

394+ let (ys1, deleted) = del x ys

395+ in if deleted

396+ then (ys1, Nothing)

397+ else (ys1, Just x)

398+ in return

399+ $ Stream.catMaybes

400+ $ fmap snd

401+ $ Stream.postscanl' g (xs, Nothing) s2

402+

403+-- | A more efficient 'deleteFirstsBy' for streams sorted in ascending order.

404+--

405+-- Both streams can be infinite.

297406--

298407-- Space: O(1)

299408--

300409-- /Unimplemented/

301−{-# INLINE deleteInStreamAscBy #-}

302−deleteInStreamAscBy :: -- (Monad m) =>

410+{-# INLINE sortedDeleteFirstsBy #-}

411+sortedDeleteFirstsBy :: -- (Monad m) =>

303412 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a

304−deleteInStreamAscBy _eq _s1 _s2 = undefined

413+sortedDeleteFirstsBy _eq _s1 _s2 = undefined

305414

306415-- XXX Remove the MonadIO constraint. We can just cache one stream and then

307416-- implement using differenceEqBy.

308417

309−-- | This essentially appends to the second stream all the occurrences of

310−-- elements in the first stream that are not already present in the second

311−-- stream.

418+-- | Returns the first stream appended with those unique elements from the

419+-- second stream that are not already present in the first stream. Note that

420+-- this is not a commutative operation unlike a set union, argument order

421+-- matters. The behavior is similar to the 'Data.List.unionBy'.

312422--

313423-- Equivalent to the following except that @s2@ is evaluated only once:

314424--

315−-- >>> unionWithStreamGenericBy eq s1 s2 = s2 `Stream.append` (Stream.deleteInStreamGenericBy eq s2 s1)

425+-- >>> unionBy eq s1 s2 = s1 `Stream.append` Stream.deleteFirstsBy eq s1 (Stream.nub s2)

316426--

317427-- Example:

318428--

319−-- >>> Stream.fold Fold.toList $ Stream.unionWithStreamGenericBy (==) (Stream.fromList [1,1,2,3]) (Stream.fromList [1,2,2,4])

429+-- >>> f s1 s2 = Stream.fold Fold.toList $ Stream.unionBy (==) (Stream.fromList s1) (Stream.fromList s2)

430+-- >>> f [1,2,2,4] [1,1,2,3,3]

320431-- [1,2,2,4,3]

321432--

433+-- First stream can be infinite, but second stream must be finite. Note that if

434+-- the first stream is infinite the union means just the first stream. Thus

435+-- union is useful only when both streams are finite. See 'sortedUnionBy' where

436+-- union can work on infinite streams if they are sorted.

437+--

322438-- Space: O(n)

323439--

324440-- Time: O(m x n)

325441--

326442-- /Pre-release/

327−{-# INLINE unionWithStreamGenericBy #-}

328−unionWithStreamGenericBy :: MonadIO m =>

443+{-# INLINE unionBy #-}

444+unionBy :: MonadIO m =>

329445 (a -> a -> Bool) -> Stream m a -> Stream m a -> Stream m a

330−unionWithStreamGenericBy eq s1 s2 =

446+unionBy eq s2 s1 =

331447 Stream.concatEffect

332448 $ do

449+ -- XXX use a rewrite rule such that if a list converted to stream

450+ -- is passed to unionBy then this becomes an identity operation.

333451 xs <- Stream.fold Fold.toList s1

334452 -- XXX we can use postscanlMAfter' instead of IORef

335453 ref <- liftIO $ newIORef $! List.nubBy eq xs

@@ -342,12 +460,21 @@ unionWithStreamGenericBy eq s1 s2 =

342460 return $ Stream.fromList xs1

343461 return $ Stream.mapM f s2 `Stream.append` s3

344462

345−-- | A more efficient 'unionWithStreamGenericBy' for sorted streams.

463+-- | A more efficient 'unionBy' for sorted streams.

464+--

465+-- Note that the behavior is different from 'unionBy'. In 'unionBy' we append

466+-- the unique elements from second stream only after exhausting the first one

467+-- whereas in sorted streams we can determine unique elements early even when

468+-- we are going through the first stream. Thus the result is an interleaving of

469+-- the two streams, merging those elements from the second stream that are not

470+-- present in the first.

346471--

347472-- Space: O(1)

348473--

474+-- Both streams can be infinite.

475+--

349476-- /Unimplemented/

350−{-# INLINE unionWithStreamAscBy #-}

351−unionWithStreamAscBy :: -- (Monad m) =>

477+{-# INLINE sortedUnionBy #-}

478+sortedUnionBy :: -- (Monad m) =>

352479 (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a

353−unionWithStreamAscBy _eq _s1 _s2 = undefined

480+sortedUnionBy _eq _s1 _s2 = undefined

core/src/Streamly/Internal/Data/Stream/Transform.hs+53−27

@@ -21,8 +21,6 @@ module Streamly.Internal.Data.Stream.Transform

2121

2222 -- * Mapping

2323 -- | Stateless one-to-one maps.

24− , map

25− , mapM

2624 , sequence

2725

2826 -- * Mapping Effects

@@ -75,7 +73,7 @@ module Streamly.Internal.Data.Stream.Transform

7573 , scanMaybe

7674 , filter

7775 , filterM

78− , deleteBy

76+ , deleteBy -- deleteOnceBy?

7977 , uniqBy

8078 , uniq

8179 , prune

@@ -83,9 +81,8 @@ module Streamly.Internal.Data.Stream.Transform

8381

8482 -- * Trimming

8583 -- | Produce a subset of the stream trimmed at ends.

86− , take

87− , takeWhile

88− , takeWhileM

84+ , initNonEmpty

85+ , tailNonEmpty

8986 , takeWhileLast

9087 , takeWhileAround

9188 , drop

@@ -132,7 +129,6 @@ module Streamly.Internal.Data.Stream.Transform

132129 -- * Searching

133130 , findIndices

134131 , elemIndices

135− , slicesBy

136132

137133 -- * Rolling map

138134 -- | Map using the previous element.

@@ -1074,6 +1070,55 @@ repeated = undefined

10741070-- Trimming

10751071------------------------------------------------------------------------------

10761072

1073+-- | init for non-empty streams, fails for empty stream case.

1074+--

1075+{-# INLINE initNonEmpty #-}

1076+initNonEmpty :: Monad m => Stream m a -> Stream m a

1077+initNonEmpty (Stream step1 state1) = Stream step (Nothing, state1)

1078+

1079+ where

1080+

1081+ step gst (Nothing, s1) = do

1082+ r <- step1 (adaptState gst) s1

1083+ return $

1084+ case r of

1085+ Yield x s -> Skip (Just x, s)

1086+ Skip s -> Skip (Nothing, s)

1087+ Stop -> error "initNonEmpty: empty Stream"

1088+

1089+ step gst (Just a, s1) = do

1090+ r <- step1 (adaptState gst) s1

1091+ return $

1092+ case r of

1093+ Yield x s -> Yield a (Just x, s)

1094+ Skip s -> Skip (Just a, s)

1095+ Stop -> Stop

1096+

1097+-- | tail for non-empty streams, fails for empty stream case.

1098+--

1099+-- See also 'tail' for a non-partial version of this function..

1100+{-# INLINE tailNonEmpty #-}

1101+tailNonEmpty :: Monad m => Stream m a -> Stream m a

1102+tailNonEmpty (Stream step1 state1) = Stream step (Nothing, state1)

1103+

1104+ where

1105+

1106+ step gst (Nothing, s1) = do

1107+ r <- step1 (adaptState gst) s1

1108+ return $

1109+ case r of

1110+ Yield x s -> Skip (Just x, s)

1111+ Skip s -> Skip (Nothing, s)

1112+ Stop -> error "tailNonEmpty: empty Stream"

1113+

1114+ step gst (Just a, s1) = do

1115+ r <- step1 (adaptState gst) s1

1116+ return $

1117+ case r of

1118+ Yield x s -> Yield x (Just x, s)

1119+ Skip s -> Skip (Just a, s)

1120+ Stop -> Stop

1121+

10771122-- | Take all consecutive elements at the end of the stream for which the

10781123-- predicate is true.

10791124--

@@ -1537,7 +1582,7 @@ reverse m = Stream step Nothing

15371582{-# INLINE reverseUnbox #-}

15381583reverseUnbox :: (MonadIO m, Unbox a) => Stream m a -> Stream m a

15391584reverseUnbox =

1540− A.flattenArraysRev -- unfoldMany A.readRev

1585+ A.concatRev -- unfoldMany A.readerRev

15411586 . fromStreamK

15421587 . K.reverse

15431588 . toStreamK

@@ -1709,25 +1754,6 @@ findIndices p (Stream step state) = Stream step' (state, 0)

17091754elemIndices :: (Monad m, Eq a) => a -> Stream m a -> Stream m Int

17101755elemIndices a = findIndices (== a)

17111756

1712−{-# INLINE_NORMAL slicesBy #-}

1713−slicesBy :: Monad m => (a -> Bool) -> Stream m a -> Stream m (Int, Int)

1714−slicesBy p (Stream step1 state1) = Stream step (Just (state1, 0, 0))

1715−

1716− where

1717−

1718− {-# INLINE_LATE step #-}

1719− step gst (Just (st, i, len)) = i `seq` len `seq` do

1720− r <- step1 (adaptState gst) st

1721− return

1722− $ case r of

1723− Yield x s ->

1724− if p x

1725− then Yield (i, len + 1) (Just (s, i + len + 1, 0))

1726− else Skip (Just (s, i, len + 1))

1727− Skip s -> Skip (Just (s, i, len))

1728− Stop -> if len == 0 then Stop else Yield (i, len) Nothing

1729− step _ Nothing = return Stop

1730−

17311757------------------------------------------------------------------------------

17321758-- Rolling map

17331759------------------------------------------------------------------------------

core/src/Streamly/Internal/Data/Stream/Transformer.hs+2−1

@@ -11,10 +11,11 @@

1111

1212module Streamly.Internal.Data.Stream.Transformer

1313 (

14+ -- * Fold to Transformer Monad

1415 foldlT

1516 , foldrT

1617

17− -- * Transform Inner Monad

18+ -- * Inner Monad Operations

1819 , liftInner

1920 , runReaderT

2021 , usingReaderT

core/src/Streamly/Internal/Data/Stream/Type.hs+132−3

@@ -1,5 +1,11 @@

11{-# LANGUAGE CPP #-}

2+{-# LANGUAGE PatternSynonyms #-}

3+{-# LANGUAGE TypeFamilies #-}

4+-- Must come after TypeFamilies, otherwise it is re-enabled.

5+-- MonoLocalBinds enabled by TypeFamilies causes perf regressions in general.

6+{-# LANGUAGE NoMonoLocalBinds #-}

27{-# LANGUAGE UndecidableInstances #-}

8+{-# LANGUAGE ViewPatterns #-}

410-- |

511-- Module : Streamly.Internal.Data.Stream.Type

@@ -16,18 +22,18 @@

1622

1723module Streamly.Internal.Data.Stream.Type

1824 (

19− -- * The stream type

25+ -- * Type

2026 Step (..)

2127 -- XXX UnStream is exported to avoid a performance issue in some

2228 -- combinators if we use the pattern synonym "Stream".

2329 , Stream (Stream, UnStream)

2430

25− -- * CrossStream type wrapper

31+ -- * CrossStream

2632 , CrossStream

2733 , unCross

2834 , mkCross

2935

30− -- * Conversion to StreamK

36+ -- * To StreamK

3137 , fromStreamK

3238 , toStreamK

3339

@@ -85,6 +91,7 @@ module Streamly.Internal.Data.Stream.Type

8591

8692 -- * Combining Two Streams

8793 -- ** Zipping

94+ -- | Zip corresponding elements of two streams.

8895 , zipWithM

8996 , zipWith

9097

@@ -100,6 +107,8 @@ module Streamly.Internal.Data.Stream.Type

100107 , unfoldMany

101108

102109 -- * Concat

110+ -- | Generate streams by mapping a stream generator on each element of an

111+ -- input stream, append the resulting streams and flatten.

103112 , concatEffect

104113 , concatMap

105114 , concatMapM

@@ -123,14 +132,22 @@ module Streamly.Internal.Data.Stream.Type

123132 , foldManyPost

124133 , groupsOf

125134 , refoldMany

135+ , refoldIterateM

126136

127137 -- * Fold Iterate

128138 , reduceIterateBfs

129139 , foldIterateBfs

130140

141+ -- * Splitting

142+ , indexOnSuffix

143+

131144 -- * Multi-stream folds

145+ -- | These should probably be expressed using zipping operations.

132146 , eqBy

133147 , cmpBy

148+

149+ -- * Deprecated

150+ , sliceOnSuffix

134151 )

135152where

136153

@@ -162,6 +179,7 @@ import Streamly.Internal.Data.Maybe.Strict (Maybe'(..), toMaybe)

162179import Streamly.Internal.Data.Refold.Type (Refold(..))

163180import Streamly.Internal.Data.Stream.Step (Step (..))

164181import Streamly.Internal.Data.SVar.Type (State, adaptState, defState)

182+import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))

165183import Streamly.Internal.Data.Unfold.Type (Unfold(..))

166184

167185import qualified Streamly.Internal.Data.Fold.Type as FL hiding (foldr)

@@ -212,6 +230,9 @@ pattern Stream step state <- (unShare -> UnStream step state)

212230nilM :: Applicative m => m b -> Stream m a

213231nilM m = Stream (\_ _ -> m $> Stop) ()

214232

233+

234+infixr 5 `consM`

235+

215236-- | Like 'cons' but fuses an effect instead of a pure value.

216237{-# INLINE_NORMAL consM #-}

217238consM :: Applicative m => m a -> Stream m a -> Stream m a

@@ -551,8 +572,12 @@ foldAdd f =

551572-- >>> Stream.foldrM step (return False) s

552573-- True

553574--

575+-- >>> import Control.Monad (join)

576+-- >>> foldrM f z = join $ Stream.foldr f z

577+--

554578{-# INLINE_NORMAL foldrM #-}

555579foldrM :: Monad m => (a -> m b -> m b) -> m b -> Stream m a -> m b

580+-- foldrM f z = join . Streamly.Internal.Data.Stream.StreamD.Type.foldr f z

556581foldrM f z (Stream step state) = go SPEC state

557582 where

558583 {-# INLINE_LATE go #-}

@@ -1971,6 +1996,110 @@ refoldMany (Refold fstep inject extract) action (Stream step state) =

19711996 step' _ (FoldManyYield b next) = return $ Yield b next

19721997 step' _ FoldManyDone = return Stop

19731998

1999+{-# ANN type CIterState Fuse #-}

2000+data CIterState s f fs b

2001+ = CIterInit s f

2002+ | CIterConsume s fs

2003+ | CIterYield b (CIterState s f fs b)

2004+ | CIterStop

2005+

2006+-- | Like 'foldIterateM' but using the 'Refold' type instead. This could be

2007+-- much more efficient due to stream fusion.

2008+--

2009+-- /Internal/

2010+{-# INLINE_NORMAL refoldIterateM #-}

2011+refoldIterateM ::

2012+ Monad m => Refold m b a b -> m b -> Stream m a -> Stream m b

2013+refoldIterateM (Refold fstep finject fextract) initial (Stream step state) =

2014+ Stream stepOuter (CIterInit state initial)

2015+

2016+ where

2017+

2018+ {-# INLINE iterStep #-}

2019+ iterStep st action = do

2020+ res <- action

2021+ return

2022+ $ Skip

2023+ $ case res of

2024+ FL.Partial fs -> CIterConsume st fs

2025+ FL.Done fb -> CIterYield fb $ CIterInit st (return fb)

2026+

2027+ {-# INLINE_LATE stepOuter #-}

2028+ stepOuter _ (CIterInit st action) = do

2029+ iterStep st (action >>= finject)

2030+ stepOuter gst (CIterConsume st fs) = do

2031+ r <- step (adaptState gst) st

2032+ case r of

2033+ Yield x s -> iterStep s (fstep fs x)

2034+ Skip s -> return $ Skip $ CIterConsume s fs

2035+ Stop -> do

2036+ b <- fextract fs

2037+ return $ Skip $ CIterYield b CIterStop

2038+ stepOuter _ (CIterYield a next) = return $ Yield a next

2039+ stepOuter _ CIterStop = return Stop

2040+

2041+-- | The refold @indexerBy f n@ takes an (index, len) tuple as initial input,

2042+-- and returns @(index + len + n, b)@ as output where @b@ is the output of the

2043+-- fold.

2044+{-# INLINE indexerBy #-}

2045+indexerBy :: Monad m =>

2046+ Fold m a Int -> Int -> Refold m (Int, Int) a (Int, Int)

2047+indexerBy (Fold step1 initial1 extract1 _final) n =

2048+ Refold step inject extract

2049+

2050+ where

2051+

2052+ inject (i, len) = do

2053+ r <- initial1

2054+ return $ case r of

2055+ FL.Partial s -> FL.Partial $ Tuple' (i + len + n) s

2056+ FL.Done l -> FL.Done (i, l)

2057+

2058+ step (Tuple' i s) x = do

2059+ r <- step1 s x

2060+ return $ case r of

2061+ FL.Partial s1 -> FL.Partial $ Tuple' i s1

2062+ FL.Done len -> FL.Done (i, len)

2063+

2064+ extract (Tuple' i s) = (i,) <$> extract1 s

2065+

2066+-- | Like 'splitOnSuffix' but generates a stream of (index, len) tuples marking

2067+-- the places where the predicate matches in the stream.

2068+--

2069+-- /Pre-release/

2070+{-# INLINE indexOnSuffix #-}

2071+indexOnSuffix :: Monad m =>

2072+ (a -> Bool) -> Stream m a -> Stream m (Int, Int)

2073+indexOnSuffix predicate =

2074+ -- Scan the stream with the given refold

2075+ refoldIterateM

2076+ (indexerBy (FL.takeEndBy_ predicate FL.length) 1)

2077+ (return (-1, 0))

2078+

2079+-- Alternate implementation

2080+{-# INLINE_NORMAL _indexOnSuffix #-}

2081+_indexOnSuffix :: Monad m => (a -> Bool) -> Stream m a -> Stream m (Int, Int)

2082+_indexOnSuffix p (Stream step1 state1) = Stream step (Just (state1, 0, 0))

2083+

2084+ where

2085+

2086+ {-# INLINE_LATE step #-}

2087+ step gst (Just (st, i, len)) = i `seq` len `seq` do

2088+ r <- step1 (adaptState gst) st

2089+ return

2090+ $ case r of

2091+ Yield x s ->

2092+ if p x

2093+ then Yield (i, len + 1) (Just (s, i + len + 1, 0))

2094+ else Skip (Just (s, i, len + 1))

2095+ Skip s -> Skip (Just (s, i, len))

2096+ Stop -> if len == 0 then Stop else Yield (i, len) Nothing

2097+ step _ Nothing = return Stop

2098+

2099+{-# DEPRECATED sliceOnSuffix "Please use indexOnSuffix instead." #-}

2100+sliceOnSuffix :: Monad m => (a -> Bool) -> Stream m a -> Stream m (Int, Int)

2101+sliceOnSuffix = indexOnSuffix

2102+

19742103------------------------------------------------------------------------------

19752104-- Stream with a cross product style monad instance

19762105------------------------------------------------------------------------------

core/src/Streamly/Internal/Data/StreamK.hs+21−29

@@ -16,16 +16,15 @@ module Streamly.Internal.Data.StreamK

1616 --

1717 -- $setup

1818

19− -- * The stream type

2019 module Streamly.Internal.Data.StreamK.Type

20+ -- * Transformer

2121 , module Streamly.Internal.Data.StreamK.Transformer

2222

23− , StreamK(..)

23+ -- * From containers

2424 , fromList

2525 , fromStream

26− , toStream

2726

28− -- ** Specialized Generation

27+ -- * Specialized Generation

2928 , repeatM

3029 , replicate

3130 , replicateM

@@ -37,8 +36,6 @@ module Streamly.Internal.Data.StreamK

3736 -- * Elimination

3837 -- ** General Folds

3938 , foldr1

40− , foldlM'

41− , foldlMx'

4239 , fold

4340 , foldBreak

4441 , foldEither

@@ -69,13 +66,13 @@ module Streamly.Internal.Data.StreamK

6966 , find

7067 , (!!)

7168

69+ -- ** To Containers

70+ , toList

71+ , toStream

72+

7273 -- ** Map and Fold

7374 , mapM_

7475

75− -- ** Conversions

76− , toList

77− , hoist

78−

7976 -- * Transformation

8077 -- ** By folding (scans)

8178 , scanl'

@@ -102,6 +99,7 @@ module Streamly.Internal.Data.StreamK

10299

103100 -- ** Reordering

104101 , sortBy

102+ , sortOn

105103

106104 -- ** Map and Filter

107105 , mapMaybe

@@ -117,6 +115,9 @@ module Streamly.Internal.Data.StreamK

117115 -- ** Transformation comprehensions

118116 , the

119117

118+ -- ** Transforming Inner Monad

119+ , hoist

120+

120121 -- * Exceptions

121122 , handle

122123

@@ -133,6 +134,7 @@ import Control.Exception (mask_, Exception)

133134import Control.Monad (void, join)

134135import Control.Monad.Catch (MonadCatch)

135136import Control.Monad.IO.Class (MonadIO(..))

137+import Data.Ord (comparing)

136138import Data.Proxy (Proxy(..))

137139import GHC.Types (SPEC(..))

138140import Streamly.Internal.Data.Array.Type (Array(..))

@@ -306,19 +308,6 @@ foldr1 step m = do

306308 yieldk a r = fmap (step p) (go a r)

307309 in foldStream defState yieldk single stp m1

308310

309−-- XXX replace the recursive "go" with explicit continuations.

310−-- | Like 'foldx', but with a monadic step function.

311−{-# INLINABLE foldlMx' #-}

312−foldlMx' :: Monad m

313− => (x -> a -> m x) -> m x -> (x -> m b) -> StreamK m a -> m b

314−foldlMx' step begin done = go begin

315− where

316− go !acc m1 =

317− let stop = acc >>= done

318− single a = acc >>= \b -> step b a >>= done

319− yieldk a r = acc >>= \b -> step b a >>= \x -> go (return x) r

320− in foldStream defState yieldk single stop m1

321−

322311-- | Fold a stream using the supplied left 'Fold' and reducing the resulting

323312-- expression strictly at each step. The behavior is similar to 'foldl''. A

324313-- 'Fold' can terminate early without consuming the full stream. See the

@@ -470,11 +459,6 @@ foldConcat

470459 Stream.Skip s -> go1 SPEC fs s

471460 Stream.Stop -> return $ Left fs

472461

473−-- | Like 'foldl'' but with a monadic step function.

474−{-# INLINE foldlM' #-}

475−foldlM' :: Monad m => (b -> a -> m b) -> m b -> StreamK m a -> m b

476−foldlM' step begin = foldlMx' step begin return

477−

478462------------------------------------------------------------------------------

479463-- Specialized folds

480464------------------------------------------------------------------------------

@@ -1331,7 +1315,8 @@ parserDone (ParserK.Failure n e) _ _ = pure $ ParserK.Error n e

13311315-- using parseBreakChunks. We can also use parseBreak as an alternative to the

13321316-- monad instance of ParserD.

13331317

1334−-- | Run a 'ParserK' over a chunked 'StreamK' and return the rest of the Stream.

1318+-- | Run a 'ParserK' over a chunked 'StreamK' and return the parse result and

1319+-- the remaining Stream.

13351320{-# INLINE_NORMAL parseBreakChunks #-}

13361321parseBreakChunks

13371322 :: (Monad m, Unbox a)

@@ -1736,3 +1721,10 @@ sortBy cmp =

17361721 . Stream.catRights -- its a non-failing backtracking parser

17371722 . Stream.parseMany (fmap (either id id) p)

17381723 . Stream.fromStreamK

1724+

1725+{-# INLINE sortOn #-}

1726+sortOn :: (Monad m, Ord b) => (a -> b) -> StreamK m a -> StreamK m a

1727+sortOn f =

1728+ fmap snd

1729+ . sortBy (comparing fst)

1730+ . fmap (\x -> let y = f x in y `seq` (y, x))

core/src/Streamly/Internal/Data/StreamK/Type.hs+80−11

@@ -1,4 +1,8 @@

11{-# LANGUAGE CPP #-}

2+{-# LANGUAGE TypeFamilies #-}

3+-- Must come after TypeFamilies, otherwise it is re-enabled.

4+-- MonoLocalBinds enabled by TypeFamilies causes perf regressions in general.

5+{-# LANGUAGE NoMonoLocalBinds #-}

26{-# LANGUAGE UndecidableInstances #-}

37-- |

48-- Module : Streamly.Internal.Data.StreamK.Type

@@ -84,6 +88,8 @@ module Streamly.Internal.Data.StreamK.Type

8488 -- ** Strict Left Folds

8589 , Streamly.Internal.Data.StreamK.Type.foldl'

8690 , foldlx'

91+ , foldlMx'

92+ , foldlM'

8793

8894 -- ** Lazy Right Folds

8995 , Streamly.Internal.Data.StreamK.Type.foldr

@@ -91,8 +97,11 @@ module Streamly.Internal.Data.StreamK.Type

9197 -- ** Specific Folds

9298 , drain

9399 , null

100+ , headNonEmpty

94101 , tail

102+ , tailNonEmpty

95103 , init

104+ , initNonEmpty

96105

97106 -- * Mapping

98107 , map

@@ -817,6 +826,24 @@ foldlx' step begin done m = get $ go m begin

817826foldl' :: Monad m => (b -> a -> b) -> b -> StreamK m a -> m b

818827foldl' step begin = foldlx' step begin id

819828

829+-- XXX replace the recursive "go" with explicit continuations.

830+-- | Like 'foldx', but with a monadic step function.

831+{-# INLINABLE foldlMx' #-}

832+foldlMx' :: Monad m

833+ => (x -> a -> m x) -> m x -> (x -> m b) -> StreamK m a -> m b

834+foldlMx' step begin done = go begin

835+ where

836+ go !acc m1 =

837+ let stop = acc >>= done

838+ single a = acc >>= \b -> step b a >>= done

839+ yieldk a r = acc >>= \b -> step b a >>= \x -> go (return x) r

840+ in foldStream defState yieldk single stop m1

841+

842+-- | Like 'foldl'' but with a monadic step function.

843+{-# INLINE foldlM' #-}

844+foldlM' :: Monad m => (b -> a -> m b) -> m b -> StreamK m a -> m b

845+foldlM' step begin = foldlMx' step begin return

846+

820847------------------------------------------------------------------------------

821848-- Specialized folds

822849------------------------------------------------------------------------------

@@ -1787,14 +1814,42 @@ iterateMWith cns step = go

17871814 !next <- s

17881815 foldStreamShared st stp sng yld (return next `cns` go (step next))

17891816

1790−{-# INLINE headPartial #-}

1791−headPartial :: Monad m => StreamK m a -> m a

1792−headPartial = foldrM (\x _ -> return x) (error "head of nil")

1817+-- | head for non-empty streams, fails for empty stream case.

1818+--

1819+{-# INLINE headNonEmpty #-}

1820+headNonEmpty :: Monad m => StreamK m a -> m a

1821+headNonEmpty = foldrM (\x _ -> return x) (error "headNonEmpty: empty stream")

1822+

1823+-- | init for non-empty streams, fails for empty stream case.

1824+--

1825+-- See also 'init' for a non-partial version of this function..

1826+{-# INLINE initNonEmpty #-}

1827+initNonEmpty :: Stream m a -> Stream m a

1828+initNonEmpty = go0

1829+

1830+ where

1831+

1832+ go0 m = mkStream $ \st yld sng stp ->

1833+ let stop = error "initNonEmpty: Empty Stream."

1834+ single _ = stp

1835+ yieldk a r = foldStream st yld sng stp (go1 a r)

1836+ in foldStream st yieldk single stop m

1837+

1838+ go1 a r = mkStream $ \st yld sng stp ->

1839+ let stop = stp

1840+ single _ = sng a

1841+ yieldk a1 r1 = yld a (go1 a1 r1)

1842+ in foldStream st yieldk single stop r

17931843

1794−{-# INLINE tailPartial #-}

1795−tailPartial :: StreamK m a -> StreamK m a

1796−tailPartial m = mkStream $ \st yld sng stp ->

1797− let stop = error "tail of nil"

1844+-- | tail for non-empty streams, fails for empty stream case.

1845+--

1846+-- See also 'tail' for a non-partial version of this function..

1847+--

1848+-- Note: this is same as "drop 1" with error on empty stream.

1849+{-# INLINE tailNonEmpty #-}

1850+tailNonEmpty :: StreamK m a -> StreamK m a

1851+tailNonEmpty m = mkStream $ \st yld sng stp ->

1852+ let stop = error "tailNonEmpty: empty stream"

17981853 single _ = stp

17991854 yieldk _ r = foldStream st yld sng stp r

18001855 in foldStream st yieldk single stop m

@@ -1859,10 +1914,10 @@ mfix f = mkStream $ \st yld sng stp ->

18591914 where

18601915

18611916 -- fix the head element of the stream

1862− xs = fix (f . headPartial)

1917+ xs = fix (f . headNonEmpty)

18631918

18641919 -- now fix the tail recursively

1865− ys = mfix (tailPartial . f)

1920+ ys = mfix (tailNonEmpty . f)

18661921

18671922-------------------------------------------------------------------------------

18681923-- Conversions

@@ -1893,6 +1948,15 @@ uncons m =

18931948 yieldk a r = pure (Just (a, r))

18941949 in foldStream defState yieldk single stop m

18951950

1951+-- Note that this is not a StreamK -> StreamK because then we cannot handle the

1952+-- empty stream case without making this a partial function.

1953+--

1954+-- See tailNonEmpty as well above.

1955+

1956+-- | Same as:

1957+--

1958+-- >>> tail = fmap (fmap snd) StreamK.uncons

1959+--

18961960{-# INLINE tail #-}

18971961tail :: Applicative m => StreamK m a -> m (Maybe (StreamK m a))

18981962tail =

@@ -1901,9 +1965,14 @@ tail =

19011965 yieldk _ r = pure $ Just r

19021966 in foldStream defState yieldk single stop

19031967

1904−-- | Extract all but the last element of the stream, if any.

1968+-- Note that this is not a StreamK -> StreamK because then we cannot handle the

1969+-- empty stream case without making this a partial function.

19051970--

1906−-- Note: This will end up buffering the entire stream.

1971+-- XXX How do we implement unsnoc? Make StreamK a monad and return the

1972+-- remaining stream as a result value in the monad?

1973+

1974+-- | Extract all but the last element of the stream, if any. This will end up

1975+-- evaluating the last element as well to find out that it is last.

19071976--

19081977-- /Pre-release/

19091978{-# INLINE init #-}

core/src/Streamly/Internal/Data/Time/Units.hs−4

@@ -1,7 +1,3 @@

1−{-# LANGUAGE DataKinds #-}

2−{-# LANGUAGE PolyKinds #-}

3−{-# LANGUAGE UnboxedTuples #-}

4−

51-- |

62-- Module : Streamly.Internal.Data.Time.Units

core/src/Streamly/Internal/Data/Unbox.hs+165−289

@@ -1,5 +1,8 @@

11{-# LANGUAGE DataKinds #-}

2−{-# LANGUAGE DefaultSignatures #-}

2+{-# LANGUAGE TypeFamilies #-}

3+-- Must come after TypeFamilies, otherwise it is re-enabled.

4+-- MonoLocalBinds enabled by TypeFamilies causes perf regressions in general.

5+{-# LANGUAGE NoMonoLocalBinds #-}

36{-# LANGUAGE UnboxedTuples #-}

47{-# LANGUAGE UndecidableInstances #-}

@@ -12,49 +15,34 @@

1215-- Portability : GHC

1316--

1417module Streamly.Internal.Data.Unbox

15− ( Unbox(..)

16− , PinnedState(..)

17− , MutableByteArray(..)

18− , touch

19− , getMutableByteArray#

20− , sizeOfMutableByteArray

21− , isPinned

22− , pin

23− , unpin

24− , newBytesAs

25− , newBytes

26− , pinnedNewBytes

27− , pinnedNewAlignedBytes

28− , nil

29− , putSliceUnsafe

30−

31− -- * Type Parser and Builder

32− , BoundedPtr (..)

18+ (

19+ -- ** Unbox type class

20+ Unbox(..)

3321

22+ -- ** Peek and poke utilities

23+ , BoundedPtr (..)

24+ -- Peek

3425 , Peeker (..)

3526 , read

3627 , readUnsafe

3728 , skipByte

3829 , runPeeker

39−

30+ -- Poke

4031 , pokeBoundedPtrUnsafe

4132 , pokeBoundedPtr

4233

43− -- * Generic Unbox instances

44− , genericSizeOf

45− , genericPeekByteIndex

46− , genericPokeByteIndex

47−

48− -- Classess used for generic deriving.

34+ -- ** Generic Deriving

4935 , PeekRep(..)

5036 , PokeRep(..)

5137 , SizeOfRep(..)

38+ , genericSizeOf

39+ , genericPeekByteIndex

40+ , genericPokeByteIndex

5241 ) where

5342

5443#include "MachDeps.h"

5544#include "ArrayMacros.h"

5645

57−import Control.Monad.IO.Class (MonadIO(..))

5846import Control.Monad (void, when)

5947import Data.Complex (Complex((:+)))

6048import Data.Functor ((<&>))

@@ -62,9 +50,6 @@ import Data.Functor.Const (Const(..))

6250import Data.Functor.Identity (Identity(..))

6351import Data.Kind (Type)

6452import Data.Proxy (Proxy (..))

65−#ifdef DEBUG

66−import Debug.Trace (trace)

67−#endif

6853import Foreign.Ptr (IntPtr(..), WordPtr(..))

6954import GHC.Base (IO(..))

7055import GHC.Fingerprint.Type (Fingerprint(..))

@@ -76,172 +61,16 @@ import GHC.Word (Word16(..), Word32(..), Word64(..), Word8(..))

7661import GHC.RTS.Flags (IoSubSystem(..))

7762#endif

7863import Streamly.Internal.Data.Builder (Builder (..))

79−import System.IO.Unsafe (unsafePerformIO)

8064

8165import GHC.Generics

8266import GHC.Exts

8367import GHC.TypeLits

8468import Prelude hiding (read)

8569

86−--------------------------------------------------------------------------------

87−-- The ArrayContents type

88−--------------------------------------------------------------------------------

89−

90−data PinnedState

91− = Pinned

92− | Unpinned

93−

94−-- XXX can use UnliftedNewtypes

95−data MutableByteArray = MutableByteArray (MutableByteArray# RealWorld)

96−

97−{-# INLINE getMutableByteArray# #-}

98−getMutableByteArray# :: MutableByteArray -> MutableByteArray# RealWorld

99−getMutableByteArray# (MutableByteArray mbarr) = mbarr

100−

101−{-# INLINE touch #-}

102−touch :: MutableByteArray -> IO ()

103−touch (MutableByteArray contents) =

104− IO $ \s -> case touch# contents s of s' -> (# s', () #)

105−

106−-- | Return the size of the array in bytes.

107−{-# INLINE sizeOfMutableByteArray #-}

108−sizeOfMutableByteArray :: MutableByteArray -> IO Int

109−sizeOfMutableByteArray (MutableByteArray arr) =

110− IO $ \s ->

111− case getSizeofMutableByteArray# arr s of

112− (# s1, i #) -> (# s1, I# i #)

113−

114−--------------------------------------------------------------------------------

115−-- Creation

116−--------------------------------------------------------------------------------

117−

118−{-# NOINLINE nil #-}

119−nil :: MutableByteArray

120−nil = unsafePerformIO $ newBytes 0

121−

122−{-# INLINE newBytes #-}

123−newBytes :: Int -> IO MutableByteArray

124−newBytes nbytes | nbytes < 0 =

125− errorWithoutStackTrace "newBytes: size must be >= 0"

126−newBytes (I# nbytes) = IO $ \s ->

127− case newByteArray# nbytes s of

128− (# s', mbarr# #) ->

129− let c = MutableByteArray mbarr#

130− in (# s', c #)

131−

132−{-# INLINE pinnedNewBytes #-}

133−pinnedNewBytes :: Int -> IO MutableByteArray

134−pinnedNewBytes nbytes | nbytes < 0 =

135− errorWithoutStackTrace "pinnedNewBytes: size must be >= 0"

136−pinnedNewBytes (I# nbytes) = IO $ \s ->

137− case newPinnedByteArray# nbytes s of

138− (# s', mbarr# #) ->

139− let c = MutableByteArray mbarr#

140− in (# s', c #)

141−

142−{-# INLINE pinnedNewAlignedBytes #-}

143−pinnedNewAlignedBytes :: Int -> Int -> IO MutableByteArray

144−pinnedNewAlignedBytes nbytes _align | nbytes < 0 =

145− errorWithoutStackTrace "pinnedNewAlignedBytes: size must be >= 0"

146−pinnedNewAlignedBytes (I# nbytes) (I# align) = IO $ \s ->

147− case newAlignedPinnedByteArray# nbytes align s of

148− (# s', mbarr# #) ->

149− let c = MutableByteArray mbarr#

150− in (# s', c #)

151−

152−{-# INLINE newBytesAs #-}

153−newBytesAs :: PinnedState -> Int -> IO MutableByteArray

154−newBytesAs Unpinned = newBytes

155−newBytesAs Pinned = pinnedNewBytes

156−

157−-------------------------------------------------------------------------------

158−-- Copying

159−-------------------------------------------------------------------------------

160−

161−-- | Put a sub range of a source array into a subrange of a destination array.

162−-- This is not safe as it does not check the bounds.

163−{-# INLINE putSliceUnsafe #-}

164−putSliceUnsafe ::

165− MonadIO m

166− => MutableByteArray

167− -> Int

168− -> MutableByteArray

169− -> Int

170− -> Int

171− -> m ()

172−putSliceUnsafe src srcStartBytes dst dstStartBytes lenBytes = liftIO $ do

70+import Streamly.Internal.Data.MutByteArray.Type (MutByteArray(..))

17371#ifdef DEBUG

174− srcLen <- sizeOfMutableByteArray src

175− dstLen <- sizeOfMutableByteArray src

176− when (srcLen - srcStartBytes < lenBytes) $ error "SRC: Insufficient length."

177− when (dstLen - dstStartBytes < lenBytes) $ error "DST: Insufficient length."

72+import Streamly.Internal.Data.MutByteArray.Type (sizeOfMutableByteArray)

17873#endif

179− let !(I# srcStartBytes#) = srcStartBytes

180− !(I# dstStartBytes#) = dstStartBytes

181− !(I# lenBytes#) = lenBytes

182− let arrS# = getMutableByteArray# src

183− arrD# = getMutableByteArray# dst

184− IO $ \s# -> (# copyMutableByteArray#

185− arrS# srcStartBytes# arrD# dstStartBytes# lenBytes# s#

186− , () #)

187−

188−-------------------------------------------------------------------------------

189−-- Pinning & Unpinning

190−-------------------------------------------------------------------------------

191−

192−{-# INLINE isPinned #-}

193−isPinned :: MutableByteArray -> Bool

194−isPinned (MutableByteArray arr#) =

195− let pinnedInt = I# (isMutableByteArrayPinned# arr#)

196− in pinnedInt /= 0

197−

198−

199−{-# INLINE cloneMutableArrayWith# #-}

200−cloneMutableArrayWith#

201− :: (Int# -> State# RealWorld -> (# State# RealWorld

202− , MutableByteArray# RealWorld #))

203− -> MutableByteArray# RealWorld

204− -> State# RealWorld

205− -> (# State# RealWorld, MutableByteArray# RealWorld #)

206−cloneMutableArrayWith# alloc# arr# s# =

207− case getSizeofMutableByteArray# arr# s# of

208− (# s1#, i# #) ->

209− case alloc# i# s1# of

210− (# s2#, arr1# #) ->

211− case copyMutableByteArray# arr# 0# arr1# 0# i# s2# of

212− s3# -> (# s3#, arr1# #)

213−

214−{-# INLINE pin #-}

215−pin :: MutableByteArray -> IO MutableByteArray

216−pin arr@(MutableByteArray marr#) =

217− if isPinned arr

218− then return arr

219− else

220−#ifdef DEBUG

221− do

222− -- XXX dump stack trace

223− trace ("pin: Copying array") (return ())

224−#endif

225− IO

226− $ \s# ->

227− case cloneMutableArrayWith# newPinnedByteArray# marr# s# of

228− (# s1#, marr1# #) -> (# s1#, MutableByteArray marr1# #)

229−

230−{-# INLINE unpin #-}

231−unpin :: MutableByteArray -> IO MutableByteArray

232−unpin arr@(MutableByteArray marr#) =

233− if not (isPinned arr)

234− then return arr

235− else

236−#ifdef DEBUG

237− do

238− -- XXX dump stack trace

239− trace ("unpin: Copying array") (return ())

240−#endif

241− IO

242− $ \s# ->

243− case cloneMutableArrayWith# newByteArray# marr# s# of

244− (# s1#, marr1# #) -> (# s1#, MutableByteArray marr1# #)

24574

24675--------------------------------------------------------------------------------

24776-- The Unbox type class

@@ -327,23 +156,37 @@ unpin arr@(MutableByteArray marr#) =

327156-- size fields at each nesting level, aggregating the size upwards.

328157

329158-- | The 'Unbox' type class provides operations for serialization (unboxing)

330−-- and deserialization (boxing) of a fixed-length, non-recursive Haskell data

331−-- type to and from its unboxed byte representation in a mutable byte array.

159+-- and deserialization (boxing) of fixed-length, non-recursive Haskell data

160+-- types to and from their byte stream representation.

161+--

332162-- Unbox uses fixed size encoding, therefore, size is independent of the value,

333−-- it can be determined solely by the type. This makes types with 'Unbox'

334−-- instances suitable for storing in arrays. Note that sum types may have

335−-- multiple constructors of different sizes, the size of the sum type is

163+-- it must be determined solely by the type. This restriction makes types with

164+-- 'Unbox' instances suitable for storing in arrays. Note that sum types may

165+-- have multiple constructors of different sizes, the size of a sum type is

336166-- computed as the maximum required by any constructor.

337167--

338−-- The 'peekByteIndex' read operation converts a Haskell type from its unboxed

339−-- representation stored in a mutable byte array, while the 'pokeByteIndex'

340−-- write operation serializes a Haskell data type to its byte representation in

341−-- the mutable byte array. These operations do not check the bounds of the

342−-- array, the user of the type class is expected to check the bounds before

343−-- peeking or poking.

344−--

345−-- Instances can be derived via Generics or Template Haskell. Note that the

346−-- data type must be non-recursive.

168+-- The 'peekAt' operation reads as many bytes from the mutable byte

169+-- array as the @size@ of the data type and builds a Haskell data type from

170+-- these bytes. 'pokeAt' operation converts a Haskell data type to its

171+-- binary representation which consists of @size@ bytes and then stores

172+-- these bytes into the mutable byte array. These operations do not check the

173+-- bounds of the array, the user of the type class is expected to check the

174+-- bounds before peeking or poking.

175+--

176+-- IMPORTANT: The serialized data's byte ordering remains the same as the host

177+-- machine's byte order. Therefore, it can not be deserialized from host

178+-- machines with a different byte ordering.

179+--

180+-- Instances can be derived via Generics, Template Haskell, or written

181+-- manually. Note that the data type must be non-recursive. WARNING! Generic

182+-- and Template Haskell deriving, both hang for recursive data types. Deriving

183+-- via Generics is more convenient but Template Haskell should be preferred

184+-- over Generics for the following reasons:

185+--

186+-- 1. Instances derived via Template Haskell provide better and more reliable

187+-- performance.

188+-- 2. Generic deriving allows only 256 fields or constructor tags whereas

189+-- template Haskell has no limit.

347190--

348191-- Here is an example, for deriving an instance of this type class using

349192-- generics:

@@ -356,113 +199,136 @@ unpin arr@(MutableByteArray marr#) =

356199-- } deriving Generic

357200-- :}

358201--

202+-- >>> import Streamly.Data.MutByteArray (Unbox(..))

203+-- >>> instance Unbox Object

204+--

359205-- To derive the instance via Template Haskell:

360206--

361207-- @

362−-- import Streamly.Data.Serialize (deriveUnbox)

363−-- $(deriveUnbox ''Object)

208+-- import Streamly.Data.MutByteArray (deriveUnbox)

209+-- \$(deriveUnbox [d|instance Unbox Object|])

364210-- @

365211--

366−-- See 'Streamly.Data.Serialize.deriveUnbox' and

367−-- 'Streamly.Data.Serialize.deriveUnboxWith' for more information on deriving

212+-- See 'Streamly.Data.MutByteArray.deriveUnbox' for more information on deriving

368213-- using Template Haskell.

369214--

370−-- Deriving via Template Haskell should be preferred for the following reasons:

371−-- 1. Instances derived via Template Haskell have more performant routines.

372−-- 2. Template Haskell deriving can handle (maxBound :: Word64) number of

373−-- constructors whereas the Generic deriving can only handle 256.

374−--

375−-- WARNING! Generic and Template Haskell deriving, both hang for recursive data

376−-- types.

377−--

378−-- >>> import Streamly.Data.Array (Unbox(..))

379−-- >>> instance Unbox Object

380−--

381215-- If you want to write the instance manually:

382216--

383217-- >>> :{

384218-- instance Unbox Object where

385219-- sizeOf _ = 16

386−-- peekByteIndex i arr = do

220+-- peekAt i arr = do

387221-- -- Check the array bounds

388−-- x0 <- peekByteIndex i arr

389−-- x1 <- peekByteIndex (i + 8) arr

222+-- x0 <- peekAt i arr

223+-- x1 <- peekAt (i + 8) arr

390224-- return $ Object x0 x1

391−-- pokeByteIndex i arr (Object x0 x1) = do

225+-- pokeAt i arr (Object x0 x1) = do

392226-- -- Check the array bounds

393−-- pokeByteIndex i arr x0

394−-- pokeByteIndex (i + 8) arr x1

227+-- pokeAt i arr x0

228+-- pokeAt (i + 8) arr x1

395229-- :}

396230--

397231class Unbox a where

398− -- | Get the size. Size cannot be zero.

232+ -- | Get the size. Size cannot be zero, should be at least 1 byte.

399233 sizeOf :: Proxy a -> Int

400234

401235 {-# INLINE sizeOf #-}

402236 default sizeOf :: (SizeOfRep (Rep a)) => Proxy a -> Int

403237 sizeOf = genericSizeOf

404238

405− -- | Read an element of type "a" from a MutableByteArray given the byte

406− -- index.

239+ -- | @peekAt byte-offset array@ reads an element of type @a@ from the

240+ -- the given the byte offset in the array.

407241 --

408242 -- IMPORTANT: The implementation of this interface may not check the bounds

409243 -- of the array, the caller must not assume that.

410− peekByteIndex :: Int -> MutableByteArray -> IO a

244+ peekAt :: Int -> MutByteArray -> IO a

245+

246+ {-# INLINE peekAt #-}

247+ default peekAt :: (Generic a, PeekRep (Rep a)) =>

248+ Int -> MutByteArray -> IO a

249+ peekAt i arr = genericPeekByteIndex arr i

411250

412− {-# INLINE peekByteIndex #-}

413− default peekByteIndex :: (Generic a, PeekRep (Rep a)) =>

414− Int -> MutableByteArray -> IO a

415− peekByteIndex i arr = genericPeekByteIndex arr i

251+ peekByteIndex :: Int -> MutByteArray -> IO a

252+ peekByteIndex = peekAt

416253

417− -- | Write an element of type "a" to a MutableByteArray given the byte

418− -- index.

254+ -- | @pokeAt byte-offset array@ writes an element of type @a@ to the

255+ -- the given the byte offset in the array.

419256 --

420257 -- IMPORTANT: The implementation of this interface may not check the bounds

421258 -- of the array, the caller must not assume that.

422− pokeByteIndex :: Int -> MutableByteArray -> a -> IO ()

259+ pokeAt :: Int -> MutByteArray -> a -> IO ()

260+

261+ pokeByteIndex :: Int -> MutByteArray -> a -> IO ()

262+ pokeByteIndex = pokeAt

423263

424− {-# INLINE pokeByteIndex #-}

425− default pokeByteIndex :: (Generic a, PokeRep (Rep a)) =>

426− Int -> MutableByteArray -> a -> IO ()

427− pokeByteIndex i arr = genericPokeByteIndex arr i

264+ {-# INLINE pokeAt #-}

265+ default pokeAt :: (Generic a, PokeRep (Rep a)) =>

266+ Int -> MutByteArray -> a -> IO ()

267+ pokeAt i arr = genericPokeByteIndex arr i

428268

429−-- XXX Add asserts to check bounds

269+{-# DEPRECATED peekByteIndex "Use peekAt." #-}

270+{-# DEPRECATED pokeByteIndex "Use pokeAt." #-}

271+

272+-- _size is the length from array start to the last accessed byte.

273+{-# INLINE checkBounds #-}

274+checkBounds :: String -> Int -> MutByteArray -> IO ()

275+checkBounds _label _size _arr = do

276+#ifdef DEBUG

277+ sz <- sizeOfMutableByteArray _arr

278+ if (_size > sz)

279+ then error

280+ $ _label

281+ ++ ": accessing array at offset = "

282+ ++ show (_size - 1)

283+ ++ " max valid offset = " ++ show (sz - 1)

284+ else return ()

285+#else

286+ return ()

287+#endif

430288

431289#define DERIVE_UNBOXED(_type, _constructor, _readArray, _writeArray, _sizeOf) \

432−instance Unbox _type where { \

433−; {-# INLINE peekByteIndex #-} \

434−; peekByteIndex (I# n) (MutableByteArray mbarr) = IO $ \s -> \

435− case _readArray mbarr n s of \

436− { (# s1, i #) -> (# s1, _constructor i #) } \

437−; {-# INLINE pokeByteIndex #-} \

438−; pokeByteIndex (I# n) (MutableByteArray mbarr) (_constructor val) = \

439− IO $ \s -> (# _writeArray mbarr n val s, () #) \

440−; {-# INLINE sizeOf #-} \

441−; sizeOf _ = _sizeOf \

290+instance Unbox _type where { \

291+; {-# INLINE peekAt #-} \

292+; peekAt off@(I# n) arr@(MutByteArray mbarr) = \

293+ checkBounds "peek" (off + sizeOf (Proxy :: Proxy _type)) arr \

294+ >> (IO $ \s -> \

295+ case _readArray mbarr n s of \

296+ { (# s1, i #) -> (# s1, _constructor i #) }) \

297+; {-# INLINE pokeAt #-} \

298+; pokeAt off@(I# n) arr@(MutByteArray mbarr) (_constructor val) = \

299+ checkBounds "poke" (off + sizeOf (Proxy :: Proxy _type)) arr \

300+ >> (IO $ \s -> (# _writeArray mbarr n val s, () #)) \

301+; {-# INLINE sizeOf #-} \

302+; sizeOf _ = _sizeOf \

442303}

443304

444305#define DERIVE_WRAPPED_UNBOX(_constraint, _type, _constructor, _innerType) \

445306instance _constraint Unbox _type where \

446−; {-# INLINE peekByteIndex #-} \

447−; peekByteIndex i arr = _constructor <$> peekByteIndex i arr \

448−; {-# INLINE pokeByteIndex #-} \

449−; pokeByteIndex i arr (_constructor a) = pokeByteIndex i arr a \

307+; {-# INLINE peekAt #-} \

308+; peekAt i arr = \

309+ checkBounds "peek" (i + sizeOf (Proxy :: Proxy _type)) arr \

310+ >> _constructor <$> peekAt i arr \

311+; {-# INLINE pokeAt #-} \

312+; pokeAt i arr (_constructor a) = \

313+ checkBounds "poke" (i + sizeOf (Proxy :: Proxy _type)) arr \

314+ >> pokeAt i arr a \

450315; {-# INLINE sizeOf #-} \

451316; sizeOf _ = SIZE_OF(_innerType)

452317

453−#define DERIVE_BINARY_UNBOX(_constraint, _type, _constructor, _innerType) \

454−instance _constraint Unbox _type where { \

455−; {-# INLINE peekByteIndex #-} \

456−; peekByteIndex i arr = \

457− peekByteIndex i arr >>= \

458− (\p1 -> peekByteIndex (i + SIZE_OF(_innerType)) arr \

459− <&> _constructor p1) \

460−; {-# INLINE pokeByteIndex #-} \

461−; pokeByteIndex i arr (_constructor p1 p2) = \

462− pokeByteIndex i arr p1 >> \

463− pokeByteIndex (i + SIZE_OF(_innerType)) arr p2 \

464−; {-# INLINE sizeOf #-} \

465−; sizeOf _ = 2 * SIZE_OF(_innerType) \

318+#define DERIVE_BINARY_UNBOX(_constraint, _type, _constructor, _innerType) \

319+instance _constraint Unbox _type where { \

320+; {-# INLINE peekAt #-} \

321+; peekAt i arr = \

322+ checkBounds "peek" (i + sizeOf (Proxy :: Proxy _type)) arr >> \

323+ peekAt i arr >>= \

324+ (\p1 -> peekAt (i + SIZE_OF(_innerType)) arr <&> _constructor p1) \

325+; {-# INLINE pokeAt #-} \

326+; pokeAt i arr (_constructor p1 p2) = \

327+ checkBounds "poke" (i + sizeOf (Proxy :: Proxy _type)) arr >> \

328+ pokeAt i arr p1 >> \

329+ pokeAt (i + SIZE_OF(_innerType)) arr p2 \

330+; {-# INLINE sizeOf #-} \

331+; sizeOf _ = 2 * SIZE_OF(_innerType) \

466332}

467333

468334-------------------------------------------------------------------------------

@@ -582,11 +448,13 @@ DERIVE_BINARY_UNBOX(,Fingerprint,Fingerprint,Word64)

582448

583449instance Unbox () where

584450

585− {-# INLINE peekByteIndex #-}

586− peekByteIndex _ _ = return ()

451+ {-# INLINE peekAt #-}

452+ peekAt i arr =

453+ checkBounds "peek ()" (i + sizeOf (Proxy :: Proxy ())) arr >> return ()

587454

588− {-# INLINE pokeByteIndex #-}

589− pokeByteIndex _ _ _ = return ()

455+ {-# INLINE pokeAt #-}

456+ pokeAt i arr _ =

457+ checkBounds "poke ()" (i + sizeOf (Proxy :: Proxy ())) arr >> return ()

590458

591459 {-# INLINE sizeOf #-}

592460 sizeOf _ = 1

@@ -594,11 +462,17 @@ instance Unbox () where

594462#if MIN_VERSION_base(4,15,0)

595463instance Unbox IoSubSystem where

596464

597− {-# INLINE peekByteIndex #-}

598− peekByteIndex i arr = toEnum <$> peekByteIndex i arr

465+ {-# INLINE peekAt #-}

466+ peekAt i arr =

467+ checkBounds

468+ "peek IoSubSystem" (i + sizeOf (Proxy :: Proxy IoSubSystem)) arr

469+ >> toEnum <$> peekAt i arr

599470

600− {-# INLINE pokeByteIndex #-}

601− pokeByteIndex i arr a = pokeByteIndex i arr (fromEnum a)

471+ {-# INLINE pokeAt #-}

472+ pokeAt i arr a =

473+ checkBounds

474+ "poke IoSubSystem" (i + sizeOf (Proxy :: Proxy IoSubSystem)) arr

475+ >> pokeAt i arr (fromEnum a)

602476

603477 {-# INLINE sizeOf #-}

604478 sizeOf _ = sizeOf (Proxy :: Proxy Int)

@@ -606,16 +480,18 @@ instance Unbox IoSubSystem where

606480

607481instance Unbox Bool where

608482

609− {-# INLINE peekByteIndex #-}

610− peekByteIndex i arr = do

611− res <- peekByteIndex i arr

483+ {-# INLINE peekAt #-}

484+ peekAt i arr = do

485+ checkBounds "peek Bool" (i + sizeOf (Proxy :: Proxy Bool)) arr

486+ res <- peekAt i arr

612487 return $ res /= (0 :: Int8)

613488

614− {-# INLINE pokeByteIndex #-}

615− pokeByteIndex i arr a =

616− if a

617− then pokeByteIndex i arr (1 :: Int8)

618− else pokeByteIndex i arr (0 :: Int8)

489+ {-# INLINE pokeAt #-}

490+ pokeAt i arr a =

491+ checkBounds "poke Bool" (i + sizeOf (Proxy :: Proxy Bool)) arr

492+ >> if a

493+ then pokeAt i arr (1 :: Int8)

494+ else pokeAt i arr (0 :: Int8)

619495

620496 {-# INLINE sizeOf #-}

621497 sizeOf _ = 1

@@ -630,7 +506,7 @@ instance Unbox Bool where

630506-- cheaper to just get the bound using the size of the array whenever needed?

631507data BoundedPtr =

632508 BoundedPtr

633− MutableByteArray -- byte array

509+ MutByteArray -- byte array

634510 Int -- current pos

635511 Int -- position after end

636512

@@ -658,7 +534,7 @@ readUnsafe = Peeker (Builder step)

658534 ++ show next

659535 ++ " end = " ++ show end

660536#endif

661− r <- peekByteIndex pos arr

537+ r <- peekAt pos arr

662538 return (r, BoundedPtr arr next end)

663539

664540{-# INLINE read #-}

@@ -675,7 +551,7 @@ read = Peeker (Builder step)

675551 $ error $ "read: reading beyond limit. next = "

676552 ++ show next

677553 ++ " end = " ++ show end

678− r <- peekByteIndex pos arr

554+ r <- peekAt pos arr

679555 return (r, BoundedPtr arr next end)

680556

681557{-# INLINE skipByte #-}

@@ -720,7 +596,7 @@ pokeBoundedPtrUnsafe a (BoundedPtr arr pos end) = do

720596 ++ show next

721597 ++ " end = " ++ show end

722598#endif

723− pokeByteIndex pos arr a

599+ pokeAt pos arr a

724600 return (BoundedPtr arr next end)

725601

726602{-# INLINE pokeBoundedPtr #-}

@@ -728,7 +604,7 @@ pokeBoundedPtr :: forall a. Unbox a => a -> BoundedPtr -> IO BoundedPtr

728604pokeBoundedPtr a (BoundedPtr arr pos end) = do

729605 let next = pos + sizeOf (Proxy :: Proxy a)

730606 when (next > end) $ error "pokeBoundedPtr writing beyond limit"

731− pokeByteIndex pos arr a

607+ pokeAt pos arr a

732608 return (BoundedPtr arr next end)

733609

734610--------------------------------------------------------------------------------

@@ -915,7 +791,7 @@ genericPokeObj a ptr = void $ genericPokeObject a ptr

915791

916792{-# INLINE genericPokeByteIndex #-}

917793genericPokeByteIndex :: (Generic a, PokeRep (Rep a)) =>

918− MutableByteArray -> Int -> a -> IO ()

794+ MutByteArray -> Int -> a -> IO ()

919795genericPokeByteIndex arr index x = do

920796 -- XXX Should we use unsafe poke?

921797#ifdef DEBUG

@@ -1021,7 +897,7 @@ genericPeekBoundedPtr = runPeeker genericPeeker

1021897

1022898{-# INLINE genericPeekByteIndex #-}

1023899genericPeekByteIndex :: (Generic a, PeekRep (Rep a)) =>

1024− MutableByteArray -> Int -> IO a

900+ MutByteArray -> Int -> IO a

1025901genericPeekByteIndex arr index = do

1026902 -- XXX Should we use unsafe peek?

1027903#ifdef DEBUG

core/src/Streamly/Internal/Data/Unbox/TH.hs+74−74

@@ -10,13 +10,11 @@

1010--

1111module Streamly.Internal.Data.Unbox.TH

1212 ( deriveUnbox

13− , deriveUnboxWith

1413

1514 -- th-helpers

1615 , DataCon(..)

1716 , DataType(..)

1817 , reifyDataType

19− , appsT

2018 ) where

2119

2220--------------------------------------------------------------------------------

@@ -25,6 +23,7 @@ module Streamly.Internal.Data.Unbox.TH

2523

2624import Data.Word (Word16, Word32, Word64, Word8)

2725import Data.Proxy (Proxy(..))

26+import Data.List (elemIndex)

2827

2928import Language.Haskell.TH

3029import Language.Haskell.TH.Syntax

@@ -69,10 +68,6 @@ tvName = elimTV id (\n _ -> n)

6968tyVarBndrName :: TyVarBndr_ flag -> Name

7069tyVarBndrName = tvName

7170

72−appsT :: Type -> [Type] -> Type

73−appsT x [] = x

74−appsT x (y:xs) = appsT (AppT x y) xs

75−

7671-- | Simplified info about a 'DataD'. Omits deriving, strictness,

7772-- kind info, and whether it's @data@ or @newtype@.

7873data DataType = DataType

@@ -246,6 +241,7 @@ mkSizeOfExpr headTy constructors =

246241mkPeekExprOne :: Int -> DataCon -> Q Exp

247242mkPeekExprOne tagSize (DataCon cname _ _ fields) =

248243 case fields of

244+ -- XXX Should we peek and check if the byte value is 0?

249245 [] -> [|pure $(conE cname)|]

250246 _ ->

251247 letE

@@ -257,7 +253,7 @@ mkPeekExprOne tagSize (DataCon cname _ _ fields) =

257253

258254 where

259255

260− peekField i = [|peekByteIndex $(varE (mkOffsetName i)) $(varE _arr)|]

256+ peekField i = [|peekAt $(varE (mkOffsetName i)) $(varE _arr)|]

261257

262258mkPeekExpr :: Type -> [DataCon] -> Q Exp

263259mkPeekExpr headTy cons =

@@ -271,7 +267,7 @@ mkPeekExpr headTy cons =

271267 doE

272268 [ bindS

273269 (varP _tag)

274− [|peekByteIndex $(varE _initialOffset) $(varE _arr)|]

270+ [|peekAt $(varE _initialOffset) $(varE _arr)|]

275271 , noBindS

276272 (caseE

277273 (sigE (varE _tag) (conT tagType))

@@ -304,7 +300,7 @@ mkPokeExprTag tagType tagVal = pokeTag

304300 where

305301

306302 pokeTag =

307− [|pokeByteIndex

303+ [|pokeAt

308304 $(varE _initialOffset)

309305 $(varE _arr)

310306 $((sigE (litE (IntegerL (fromIntegral tagVal))) (conT tagType)))|]

@@ -322,7 +318,7 @@ mkPokeExprFields tagSize fields = do

322318

323319 numFields = length fields

324320 pokeField i =

325− [|pokeByteIndex

321+ [|pokeAt

326322 $(varE (mkOffsetName i))

327323 $(varE _arr)

328324 $(varE (mkFieldName i))|]

@@ -341,7 +337,9 @@ mkPokeExpr headTy cons =

341337 [|error

342338 ("Attempting to poke type with no constructors (" ++

343339 $(lift (pprint headTy)) ++ ")")|]

344− [(DataCon _ _ _ [])] -> [|pure ()|]

340+ -- XXX We don't gaurentee encoded equivalilty for Unbox. Does it still

341+ -- make sense to encode a default value for unit constructor?

342+ [(DataCon _ _ _ [])] -> [|pure ()|] -- mkPokeExprTag ''Word8 0

345343 [(DataCon cname _ _ fields)] ->

346344 caseE

347345 (varE _val)

@@ -397,8 +395,8 @@ mkPokeExpr headTy cons =

397395-- [(Nothing, (ConT ''Bool)), (Nothing, (VarT (mkName "b")))]

398396-- ])

399397-- @

400−deriveUnboxInternal :: Cxt -> Type -> [DataCon] -> Q [Dec]

401−deriveUnboxInternal preds headTy cons = do

398+deriveUnboxInternal :: Type -> [DataCon] -> ([Dec] -> Q [Dec]) -> Q [Dec]

399+deriveUnboxInternal headTy cons mkDec = do

402400 sizeOfMethod <- mkSizeOfExpr headTy cons

403401 peekMethod <- mkPeekExpr headTy cons

404402 pokeMethod <- mkPokeExpr headTy cons

@@ -407,9 +405,9 @@ deriveUnboxInternal preds headTy cons = do

407405 -- none

408406 [ -- PragmaD (InlineP 'sizeOf Inline FunLike AllPhases)

409407 FunD 'sizeOf [Clause [WildP] (NormalB sizeOfMethod) []]

410− , PragmaD (InlineP 'peekByteIndex Inline FunLike AllPhases)

408+ , PragmaD (InlineP 'peekAt Inline FunLike AllPhases)

411409 , FunD

412− 'peekByteIndex

410+ 'peekAt

413411 [ Clause

414412 (if isUnitType cons

415413 then [WildP, WildP]

@@ -417,9 +415,9 @@ deriveUnboxInternal preds headTy cons = do

417415 (NormalB peekMethod)

418416 []

419417 ]

420− , PragmaD (InlineP 'pokeByteIndex Inline FunLike AllPhases)

418+ , PragmaD (InlineP 'pokeAt Inline FunLike AllPhases)

421419 , FunD

422− 'pokeByteIndex

420+ 'pokeAt

423421 [ Clause

424422 (if isUnitType cons

425423 then [WildP, WildP, WildP]

@@ -428,71 +426,73 @@ deriveUnboxInternal preds headTy cons = do

428426 []

429427 ]

430428 ]

431− return [InstanceD Nothing preds (AppT (ConT ''Unbox) headTy) methods]

429+ mkDec methods

432430

433−-- | Template haskell helper to create instances of 'Unbox' automatically.

434−--

435−-- Consider the datatype:

436−-- @

437−-- data CustomDataType a b c = ...

438−-- @

431+-- | Given an 'Unbox' instance declaration splice without the methods (e.g.

432+-- @[d|instance Unbox a => Unbox (Maybe a)|]@), generate an instance

433+-- declaration including all the type class method implementations.

439434--

440−-- Usage: @$(deriveUnbox ''CustomDataType)@

435+-- Usage:

441436--

442−-- Note: All type variables automatcally get an "Unbox" constraint.

443−-- The derived code will look like the following,

444437-- @

445−-- instance (Unbox a, Unbox b, Unbox c) => Unbox (CustomDataType a b c) where

446−-- ...

438+-- \$(deriveUnbox [d|instance Unbox a => Unbox (Maybe a)|])

447439-- @

448−--

449−-- To control which type variables get the Unbox constraint, use

450−-- 'deriveUnboxWith'.

451−deriveUnbox :: Name -> Q [Dec]

452−deriveUnbox name = do

453− dt <- reifyDataType name

454− let preds = map (unboxPred . VarT) (dtTvs dt)

455− headTy = appsT (ConT name) (map VarT (dtTvs dt))

456− cons = dtCons dt

457− deriveUnboxInternal preds headTy cons

440+deriveUnbox :: Q [Dec] -> Q [Dec]

441+deriveUnbox mDecs = do

442+ dec <- mDecs

443+ case dec of

444+ [InstanceD mo preds headTyWC []] -> do

445+ let headTy = unwrap dec headTyWC

446+ (mainTyName, subs) = getMainTypeName dec headTy

447+ dt <- reifyDataType mainTyName

448+ let tyVars = dtTvs dt

449+ mapper = mapperWith (VarT <$> tyVars) subs

450+ cons = map (modifyConVariables mapper) (dtCons dt)

451+ deriveUnboxInternal headTy cons (mkInst mo preds headTyWC)

452+ _ -> errorMessage dec

458453

459454 where

460455

461− unboxPred ty =

462−#if MIN_VERSION_template_haskell(2,10,0)

463− AppT (ConT ''Unbox) ty

464−#else

465− ClassP ''Unbox [ty]

466−#endif

456+ mapperWith l1 l2 a =

457+ case elemIndex a l1 of

458+ Nothing -> a

459+ -- XXX Capture this case and give a relavant error.

460+ Just i -> l2 !! i

461+

462+ mapType f (AppT t1 t2) = AppT (mapType f t1) (mapType f t2)

463+ mapType f (InfixT t1 n t2) = InfixT (mapType f t1) n (mapType f t2)

464+ mapType f (UInfixT t1 n t2) = UInfixT (mapType f t1) n (mapType f t2)

465+ mapType f (ParensT t) = ParensT (mapType f t)

466+ mapType f v = f v

467+

468+ modifyConVariables f con =

469+ con { dcFields = map (\(a, b) -> (a, mapType f b)) (dcFields con) }

470+

471+ mkInst mo preds headTyWC methods =

472+ pure [InstanceD mo preds headTyWC methods]

473+

474+ errorMessage dec =

475+ error $ unlines

476+ [ "Error: deriveUnbox:"

477+ , ""

478+ , ">> " ++ pprint dec

479+ , ""

480+ , "The supplied declaration not a valid instance declaration."

481+ , "Provide a valid Haskell instance declaration without a body."

482+ , ""

483+ , "Examples:"

484+ , "instance Unbox (Proxy a)"

485+ , "instance Unbox a => Unbox (Identity a)"

486+ , "instance Unbox (TableT Identity)"

487+ ]

467488

468−-- | Like 'deriveUnbox' but control which type variables get the 'Unbox'

469−-- constraint.

470−--

471−-- Consider the datatype:

472−-- @

473−-- data CustomDataType a b c = ...

474−-- @

475−--

476−-- Usage: @$(deriveUnboxWith ["a", "c"] ''CustomDataType)@

477−--

478−-- @

479−-- instance (Unbox a, Unbox c) => Unbox (CustomDataType a b c) where

480−-- ...

481−-- @

482−--

483−deriveUnboxWith :: [String] -> Name -> Q [Dec]

484−deriveUnboxWith vars name = do

485− dt <- reifyDataType name

486− let preds = map (unboxPred . VarT) (fmap mkName vars)

487− headTy = appsT (ConT name) (map VarT (dtTvs dt))

488− cons = dtCons dt

489− deriveUnboxInternal preds headTy cons

489+ unwrap _ (AppT (ConT _) r) = r

490+ unwrap dec _ = errorMessage dec

490491

491− where

492+ getMainTypeName dec = go []

492493

493− unboxPred ty =

494−#if MIN_VERSION_template_haskell(2,10,0)

495− AppT (ConT ''Unbox) ty

496−#else

497− ClassP ''Unbox [ty]

498−#endif

494+ where

495+

496+ go xs (ConT nm) = (nm, xs)

497+ go xs (AppT l r) = go (r:xs) l

498+ go _ _ = errorMessage dec

core/src/Streamly/Internal/Data/Unfold.hs+6−4

@@ -82,7 +82,7 @@ module Streamly.Internal.Data.Unfold

8282 , dropWhileM

8383

8484 -- ** Cross product

85− , joinInnerGeneric

85+ , innerJoin

8686

8787 -- ** Resource Management

8888 -- | 'bracket' is the most general resource management operation, all other

@@ -677,10 +677,12 @@ dropWhileM f (Unfold step inject) = Unfold step' inject'

677677dropWhile :: Monad m => (b -> Bool) -> Unfold m a b -> Unfold m a b

678678dropWhile f = dropWhileM (return . f)

679679

680−{-# INLINE_NORMAL joinInnerGeneric #-}

681−joinInnerGeneric :: Monad m =>

680+-- | Cross intersection of two unfolds. See

681+-- 'Streamly.Internal.Data.Stream.innerJoin' for more details.

682+{-# INLINE_NORMAL innerJoin #-}

683+innerJoin :: Monad m =>

682684 (b -> c -> Bool) -> Unfold m a b -> Unfold m a c -> Unfold m a (b, c)

683−joinInnerGeneric eq s1 s2 = filter (\(a, b) -> a `eq` b) $ cross s1 s2

685+innerJoin eq s1 s2 = filter (\(a, b) -> a `eq` b) $ cross s1 s2

684686

685687------------------------------------------------------------------------------

686688-- Exceptions

core/src/Streamly/Internal/FileSystem/File.hs+4−9

@@ -103,15 +103,14 @@ import qualified Control.Monad.Catch as MC

103103import qualified System.IO as SIO

104104

105105import Streamly.Data.Fold (groupsOf, drain)

106−import Streamly.Internal.Data.Array.Type (Array(..), pinnedWriteNUnsafe)

106+import Streamly.Internal.Data.Array.Type (Array(..))

107107import Streamly.Internal.Data.Fold.Type (Fold(..))

108108import Streamly.Data.Stream (Stream)

109−import Streamly.Internal.Data.Unbox (Unbox)

110109import Streamly.Internal.Data.Unfold.Type (Unfold(..))

111110-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)

112111import Streamly.Internal.System.IO (defaultChunkSize)

113112

114−import qualified Streamly.Data.Array as A

113+import qualified Streamly.Internal.Data.Array as A

115114import qualified Streamly.Data.Stream as S

116115import qualified Streamly.Data.Unfold as UF

117116import qualified Streamly.Internal.Data.Array.Type as IA (pinnedChunksOf)

@@ -339,10 +338,6 @@ readWithBufferOf = readerWith

339338reader :: (MonadIO m, MonadCatch m) => Unfold m FilePath Word8

340339reader = UF.many A.reader (usingFile FH.chunkReader)

341340

342−{-# INLINE concatChunks #-}

343−concatChunks :: (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

344−concatChunks = S.unfoldMany A.reader

345−

346341-- | Generate a stream of bytes from a file specified by path. The stream ends

347342-- when EOF is encountered. File is locked using multiple reader and single

348343-- writer locking mode.

@@ -351,7 +346,7 @@ concatChunks = S.unfoldMany A.reader

351346--

352347{-# INLINE read #-}

353348read :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8

354−read file = concatChunks $ withFile file ReadMode FH.readChunks

349+read file = A.concat $ withFile file ReadMode FH.readChunks

355350

356351{-# DEPRECATED toBytes "Please use 'read' instead" #-}

357352{-# INLINE toBytes #-}

@@ -467,7 +462,7 @@ writeChunks path = Fold step initial extract final

467462writeWith :: (MonadIO m, MonadCatch m)

468463 => Int -> FilePath -> Fold m Word8 ()

469464writeWith n path =

470− groupsOf n (pinnedWriteNUnsafe n) (writeChunks path)

465+ groupsOf n (A.unsafePinnedCreateOf n) (writeChunks path)

471466

472467{-# DEPRECATED writeWithBufferOf "Please use 'writeWith' instead" #-}

473468{-# INLINE writeWithBufferOf #-}

core/src/Streamly/Internal/FileSystem/Handle.hs+21−28

@@ -40,8 +40,8 @@ module Streamly.Internal.FileSystem.Handle

4040

4141 -- * Streams

4242 , read

43− , readWith

44− , readChunksWith

43+ , readWith -- readConcatChunksOf

44+ , readChunksWith -- readChunksOf

4545 , readChunks

4646

4747 -- * Unfolds

@@ -59,10 +59,10 @@ module Streamly.Internal.FileSystem.Handle

5959 -- , writeUtf8ByLines

6060 -- , writeByFrames

6161 -- , writeLines

62− , writeWith

62+ , writeWith -- writeChunksOf

6363 , writeChunks

64− , writeChunksWith

65− , writeMaybesWith

64+ , writeChunksWith -- writeCompactChunksOf

65+ , writeMaybesWith -- writeCompactJustsOf

6666

6767 -- * Refolds

6868 , writer

@@ -131,16 +131,13 @@ import Streamly.Internal.Data.Fold (Fold)

131131import Streamly.Internal.Data.Refold.Type (Refold(..))

132132import Streamly.Internal.Data.Unfold.Type (Unfold(..))

133133import Streamly.Internal.Data.Array.Type

134− (Array(..), pinnedWriteNUnsafe, unsafeFreezeWithShrink, byteLength)

134+ (Array(..), unsafeFreezeWithShrink, byteLength)

135135import Streamly.Internal.Data.Stream.Type (Stream)

136−import Streamly.Internal.Data.Stream.Chunked (lpackArraysChunksOf)

137136-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)

138137import Streamly.Internal.System.IO (defaultChunkSize)

139138

140139import qualified Streamly.Data.Fold as FL

141−import qualified Streamly.Data.Array as A

142−import qualified Streamly.Internal.Data.Array.Type as A

143−import qualified Streamly.Internal.Data.Stream.Chunked as AS

140+import qualified Streamly.Internal.Data.Array as A

144141import qualified Streamly.Internal.Data.MutArray.Type as MArray

145142import qualified Streamly.Internal.Data.Refold.Type as Refold

146143import qualified Streamly.Internal.Data.Fold.Type as FL(refoldMany)

@@ -169,6 +166,8 @@ import qualified Streamly.Internal.Data.StreamK.Type as K (mkStream)

169166-- Array IO (Input)

170167-------------------------------------------------------------------------------

171168

169+-- XXX add an API that compacts the arrays to an exact size.

170+

172171-- | Read a 'ByteArray' consisting of one or more bytes from a file handle. If

173172-- no data is available on the handle it blocks until at least one byte becomes

174173-- available. If any data is available then it immediately returns that data

@@ -178,9 +177,9 @@ import qualified Streamly.Internal.Data.StreamK.Type as K (mkStream)

178177{-# INLINABLE getChunk #-}

179178getChunk :: MonadIO m => Int -> Handle -> m (Array Word8)

180179getChunk size h = liftIO $ do

181− arr <- MArray.pinnedNewBytes size

180+ arr :: MArray.MutArray Word8 <- MArray.pinnedEmptyOf size

182181 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))

183− MArray.asPtrUnsafe arr $ \p -> do

182+ MArray.unsafePinnedAsPtr arr $ \p -> do

184183 n <- hGetBufSome h p size

185184 -- XXX shrink only if the diff is significant

186185 return $

@@ -338,10 +337,6 @@ readerWith = UF.many A.reader chunkReaderWith

338337readWithBufferOf :: MonadIO m => Unfold m (Int, Handle) Word8

339338readWithBufferOf = readerWith

340339

341−{-# INLINE concatChunks #-}

342−concatChunks :: (Monad m, Unbox a) => Stream m (Array a) -> Stream m a

343−concatChunks = S.unfoldMany A.reader

344−

345340-- | @readWith bufsize handle@ reads a byte stream from a file

346341-- handle, reads are performed in chunks of up to @bufsize@.

347342--

@@ -350,7 +345,7 @@ concatChunks = S.unfoldMany A.reader

350345-- /Pre-release/

351346{-# INLINE readWith #-}

352347readWith :: MonadIO m => Int -> Handle -> Stream m Word8

353−readWith size h = concatChunks $ readChunksWith size h

348+readWith size h = A.concat $ readChunksWith size h

354349

355350-- | Unfolds a file handle into a byte stream. IO requests to the device are

356351-- performed in sizes of

@@ -369,7 +364,7 @@ reader = UF.many A.reader chunkReader

369364-- /Pre-release/

370365{-# INLINE read #-}

371366read :: MonadIO m => Handle -> Stream m Word8

372−read = concatChunks . readChunks

367+read = A.concat . readChunks

373368

374369-------------------------------------------------------------------------------

375370-- Writing

@@ -384,12 +379,12 @@ read = concatChunks . readChunks

384379{-# INLINABLE putChunk #-}

385380putChunk :: MonadIO m => Handle -> Array a -> m ()

386381putChunk _ arr | byteLength arr == 0 = return ()

387−putChunk h arr = A.asPtrUnsafe arr $ \ptr ->

382+putChunk h arr = A.unsafePinnedAsPtr arr $ \ptr ->

388383 liftIO $ hPutBuf h ptr aLen

389384

390385 where

391386

392− -- XXX We should have the length passed by asPtrUnsafe itself.

387+ -- XXX We should have the length passed by unsafePinnedAsPtr itself.

393388 aLen = A.byteLength arr

394389

395390-------------------------------------------------------------------------------

@@ -421,7 +416,7 @@ putChunks h = S.fold (FL.drainMapM (putChunk h))

421416{-# INLINE putChunksWith #-}

422417putChunksWith :: (MonadIO m, Unbox a)

423418 => Int -> Handle -> Stream m (Array a) -> m ()

424−putChunksWith n h xs = putChunks h $ AS.compact n xs

419+putChunksWith n h xs = putChunks h $ A.compactLE n xs

425420

426421-- > putBytesWith n h m = Handle.putChunks h $ A.pinnedChunksOf n m

427422

@@ -461,8 +456,6 @@ writeChunks h = FL.drainMapM (putChunk h)

461456chunkWriter :: MonadIO m => Refold m Handle (Array a) ()

462457chunkWriter = Refold.drainBy putChunk

463458

464−-- XXX lpackArraysChunksOf should be written idiomatically

465−

466459-- | @writeChunksWith bufsize handle@ writes a stream of arrays

467460-- to @handle@ after coalescing the adjacent arrays in chunks of @bufsize@.

468461-- We never split an array, if a single array is bigger than the specified size

@@ -472,7 +465,7 @@ chunkWriter = Refold.drainBy putChunk

472465{-# INLINE writeChunksWith #-}

473466writeChunksWith :: (MonadIO m, Unbox a)

474467 => Int -> Handle -> Fold m (Array a) ()

475−writeChunksWith n h = lpackArraysChunksOf n (writeChunks h)

468+writeChunksWith n h = A.lCompactGE n (writeChunks h)

476469

477470-- | Same as 'writeChunksWith'

478471--

@@ -496,11 +489,11 @@ writeChunksWithBufferOf = writeChunksWith

496489-- Bytes in the input stream are collected into a buffer until we have a chunk

497490-- of @reqSize@ and then written to the IO device.

498491--

499−-- >>> writeWith n h = Fold.groupsOf n (Array.writeNUnsafe n) (Handle.writeChunks h)

492+-- >>> writeWith n h = Fold.groupsOf n (Array.unsafeCreateOf n) (Handle.writeChunks h)

500493--

501494{-# INLINE writeWith #-}

502495writeWith :: MonadIO m => Int -> Handle -> Fold m Word8 ()

503−writeWith n h = FL.groupsOf n (pinnedWriteNUnsafe n) (writeChunks h)

496+writeWith n h = FL.groupsOf n (A.unsafePinnedCreateOf n) (writeChunks h)

504497

505498-- | Same as 'writeWith'

506499--

@@ -518,7 +511,7 @@ writeWithBufferOf = writeWith

518511writeMaybesWith :: (MonadIO m )

519512 => Int -> Handle -> Fold m (Maybe Word8) ()

520513writeMaybesWith n h =

521− let writeNJusts = FL.lmap fromJust $ A.pinnedWriteN n

514+ let writeNJusts = FL.lmap fromJust $ A.pinnedCreateOf n

522515 writeOnNothing = FL.takeEndBy_ isNothing writeNJusts

523516 in FL.many writeOnNothing (writeChunks h)

524517

@@ -528,7 +521,7 @@ writeMaybesWith n h =

528521{-# INLINE writerWith #-}

529522writerWith :: MonadIO m => Int -> Refold m Handle Word8 ()

530523writerWith n =

531− FL.refoldMany (FL.take n $ pinnedWriteNUnsafe n) chunkWriter

524+ FL.refoldMany (FL.take n $ A.unsafePinnedCreateOf n) chunkWriter

532525

533526-- | Write a byte stream to a file handle. Accumulates the input in chunks of

534527-- up to 'Streamly.Internal.Data.Array.Type.defaultChunkSize' before writing

core/src/Streamly/Internal/FileSystem/Path.hsnew file+515

@@ -0,0 +1,515 @@

1+{-# LANGUAGE TemplateHaskell #-}

3+-- |

4+-- Module : Streamly.Internal.FileSystem.Path

6+-- License : BSD3

7+-- Maintainer : streamly@composewell.com

8+-- Portability : GHC

9+--

10+-- Well typed and flexible file systems paths, preserving the OS and filesystem

11+-- encoding.

12+--

13+-- You can choose the level of type safety you want. 'Path' is the basic path

14+-- type which can represent a file, directory, absolute or relative path with

15+-- no restrictions. Depending on how much type safety you want you can choose

16+-- appropriate type wrappers to wrap 'Path'. @File Path@ mandates the path to

17+-- be a file whereas @Abs (File Path)@ mandates it to be an absolute path

18+-- representing a file.

19+--

20+-- You can upgrade or downgrade the safety. Whenever a less restrictive path

21+-- type is converted to a more restrctive path type the conversion involves

22+-- checks and it may fail. However, a more restrictive path type can be freely

23+-- converted to a less restrictive one.

24+--

25+-- See the @streamly-filepath@ package for interworking with the 'OsPath' type.

26+-- The 'Path' type can be converted to and from 'OsPath' type at zero cost

27+-- since the underlying representation of both is the same.

28+

29+-- Conventions: A trailing separator on a path indicates that it is a directory.

30+-- However, the absence of a trailing separator does not convey any

31+-- information, it could either be a directory or a file.

32+

33+-- You may also find the 'str' quasiquoter from "Streamly.Unicode.String" to be

34+-- useful in creating paths.

35+--

36+module Streamly.Internal.FileSystem.Path

37+ (

38+ -- * Path Types

39+ Path (..)

40+ , File

41+ , Dir

42+ , Abs

43+ , Rel

44+

45+ -- * Conversions

46+ , IsPath (..)

47+ , adaptPath

48+

49+ -- * Construction

50+ , fromChunk

51+ , fromChunkUnsafe

52+ , fromString

53+ , fromChars

54+

55+ -- * Statically Verified Literals

56+ -- quasiquoters

57+ , path

58+ , abs

59+ , rel

60+ , dir

61+ , file

62+ , absdir

63+ , reldir

64+ , absfile

65+ , relfile

66+

67+ -- * Statically Verified Strings

68+ -- TH macros

69+ , mkPath

70+ , mkAbs

71+ , mkRel

72+ , mkDir

73+ , mkFile

74+ , mkAbsDir

75+ , mkRelDir

76+ , mkAbsFile

77+ , mkRelFile

78+

79+ -- * Elimination

80+ , toChunk

81+ , toString

82+ , toChars

83+

84+ -- * Operations

85+ -- Do we need to export the separator functions? They are not essential if

86+ -- operations to split and combine paths are provided. If someone wants to

87+ -- work on paths at low level then they know what they are.

88+ , primarySeparator

89+ , isSeparator

90+ , extendPath

91+ , extendDir

92+ )

93+where

94+

95+import Control.Exception (Exception)

96+import Control.Monad.Catch (MonadThrow(..))

97+import Data.Char (chr)

98+import Data.Functor.Identity (Identity(..))

99+import Data.Word (Word8)

100+#if defined(mingw32_HOST_OS) || defined(__MINGW32__)

101+import Data.Word (Word16)

102+#endif

103+import Language.Haskell.TH (Q, Exp)

104+import Language.Haskell.TH.Quote (QuasiQuoter)

105+import Streamly.Internal.Data.Array (Array)

106+import Streamly.Internal.Data.Stream (Stream)

107+import System.IO.Unsafe (unsafePerformIO)

108+

109+import qualified Streamly.Internal.Data.Array as Array

110+import qualified Streamly.Internal.Data.Fold as Fold

111+import qualified Streamly.Internal.Data.MutArray as MutArray

112+import qualified Streamly.Internal.Data.Stream as Stream

113+import qualified Streamly.Internal.Unicode.Stream as Unicode

114+

115+import Prelude hiding (abs)

116+

117+#if defined(mingw32_HOST_OS) || defined(__MINGW32__)

118+#define WORD_TYPE Word16

119+#define SEPARATOR 92

120+#else

121+#define WORD_TYPE Word8

122+#define SEPARATOR 47

123+#endif

124+

125+------------------------------------------------------------------------------

126+-- Exceptions

127+------------------------------------------------------------------------------

128+

129+-- | Exceptions thrown by path operations.

130+data PathException =

131+ InvalidPath String

132+ | InvalidAbsPath String

133+ | InvalidRelPath String

134+ | InvalidFilePath String

135+ | InvalidDirPath String

136+ deriving (Show,Eq)

137+

138+instance Exception PathException

139+

140+-- XXX Path must not contain null char on Posix. System calls consider the path

141+-- as null terminated.

142+-- XXX Maintain the Array with null termination because Unix system calls

143+-- require a null terminated string, also they return null terminated strings

144+-- as paths. Implementation of path append will have to handle the null

145+-- termination. Or we can choose to always copy the array when using it in

146+-- system calls.

147+

148+-- XXX The eq instance needs to make sure that the paths are equivalent. If we

149+-- normalize the paths we can do a byte comparison. However, on windows paths

150+-- are case insensitive but the case is preserved, therefore, we cannot

151+-- normalize and need to do case insensitive comparison.

152+

153+------------------------------------------------------------------------------

154+-- Types

155+------------------------------------------------------------------------------

156+

157+-- | A type representing file system paths for directories or files.

158+newtype Path = Path (Array WORD_TYPE) -- deriving Eq

159+

160+-- Show instance prints raw bytes without any decoding for rountdtripping.

161+-- Should we print this as a string instead, may be useful for ascii chars but

162+-- utf8 encoded chars may be unprintable. Better use toString if you want to

163+-- pretty print the path.

164+{-

165+instance Show Path where

166+ show (Path x) = show x

167+-}

168+

169+-- XXX Do we need a type for file or dir Name as names cannot have the

170+-- separator char and there may be other restrictions on names? For example,

171+-- length restriction. A file name cannot be "." or "..". We can use the types

172+-- "File Name" and "Dir Name" to represent names. Also, file systems may put

173+-- limits on names. Can have an IsName type class with members Name, (File

174+-- Name), (Dir Name).

175+

176+-- | A type representing a file path.

177+newtype File a = File a

178+

179+-- | A type representing a directory path.

180+newtype Dir a = Dir a

181+

182+-- | A type representing absolute paths.

183+newtype Abs a = Abs a

184+

185+-- | A type representing relative paths.

186+newtype Rel a = Rel a

187+

188+------------------------------------------------------------------------------

189+-- Conversions

190+------------------------------------------------------------------------------

191+

192+-- | A member of 'IsPath' knows how to convert to and from the 'Path' type.

193+class IsPath a where

194+ -- | Like 'fromPath' but does not check the properties of 'Path'. Provides

195+ -- performance and simplicity when we know that the properties of the path

196+ -- are already verified, for example, when we get the path from the file

197+ -- system or the OS APIs.

198+ fromPathUnsafe :: Path -> a

199+

200+ -- | Convert a raw 'Path' to other forms of well-typed paths. It may fail

201+ -- if the path does not satisfy the properties of the target type.

202+ --

203+ -- Path components may have limits.

204+ -- Total path length may have a limit.

205+ fromPath :: MonadThrow m => Path -> m a

206+

207+ -- | Convert a well-typed path to a raw 'Path'. Never fails.

208+ toPath :: a -> Path

209+

210+instance IsPath Path where

211+ fromPathUnsafe = id

212+ fromPath = pure

213+ toPath = id

214+

215+instance IsPath (File Path) where

216+ fromPathUnsafe p = File p

217+ fromPath p = pure (File p)

218+ toPath (File p) = p

219+

220+instance IsPath (Dir Path) where

221+ fromPathUnsafe p = Dir p

222+ fromPath p = pure (Dir p)

223+ toPath (Dir p) = p

224+

225+instance IsPath (Abs Path) where

226+ fromPathUnsafe p = Abs p

227+ fromPath p = pure (Abs p)

228+ toPath (Abs p) = p

229+

230+instance IsPath (Rel Path) where

231+ fromPathUnsafe p = Rel p

232+ fromPath p = pure (Rel p)

233+ toPath (Rel p) = p

234+

235+instance IsPath (Abs (File Path)) where

236+ fromPathUnsafe p = Abs (File p)

237+ fromPath p = pure (Abs (File p))

238+ toPath (Abs (File p)) = p

239+

240+instance IsPath (Abs (Dir Path)) where

241+ fromPathUnsafe p = Abs (Dir p)

242+ fromPath p = pure (Abs (Dir p))

243+ toPath (Abs (Dir p)) = p

244+

245+instance IsPath (Rel (File Path)) where

246+ fromPathUnsafe p = Rel (File p)

247+ fromPath p = pure (Rel (File p))

248+ toPath (Rel (File p)) = p

249+

250+instance IsPath (Rel (Dir Path)) where

251+ fromPathUnsafe p = Rel (Dir p)

252+ fromPath p = pure (Rel (Dir p))

253+ toPath (Rel (Dir p)) = p

254+

255+-- XXX Use rewrite rules to eliminate intermediate conversions for better

256+-- efficiency.

257+

258+-- | Convert a path type to another path type. This operation may fail with a

259+-- 'PathException' when converting a less restrictive path type to a more

260+-- restrictive one.

261+adaptPath :: (MonadThrow m, IsPath a, IsPath b) => a -> m b

262+adaptPath p = fromPath $ toPath p

263+

264+------------------------------------------------------------------------------

265+-- Construction

266+------------------------------------------------------------------------------

267+

268+-- A chunk is essentially an untyped Array i.e. Array Word8. We can either use

269+-- the term ByteArray for that or just Chunk. The latter is shorter and we have

270+-- been using it consistently in streamly.

271+

272+-- | /Unsafe/: On Posix, a path cannot contain null characters. On Windows, the

273+-- array passed must be a multiple of 2 bytes as the underlying representation

274+-- uses 'Word16'.

275+{-# INLINE fromChunkUnsafe #-}

276+fromChunkUnsafe :: Array Word8 -> Path

277+fromChunkUnsafe arr = Path (Array.castUnsafe arr)

278+

279+-- | On Posix it may fail if the byte array contains null characters. On

280+-- Windows the array passed must be a multiple of 2 bytes as the underlying

281+-- representation uses 'Word16'.

282+--

283+-- Throws 'InvalidPath'.

284+fromChunk :: MonadThrow m => Array Word8 -> m Path

285+fromChunk arr =

286+ case Array.cast arr of

287+ Nothing ->

288+ -- XXX Windows only message.

289+ throwM

290+ $ InvalidPath

291+ $ "Encoded path length " ++ show (Array.byteLength arr)

292+ ++ " is not a multiple of 16-bit."

293+ Just x -> pure (Path x)

294+

295+-- | Convert 'Path' to an array of bytes.

296+toChunk :: Path -> Array Word8

297+toChunk (Path arr) = Array.asBytes arr

298+

299+-- | Encode a Unicode char stream to 'Path' using strict UTF-8 encoding on

300+-- Posix. On Posix it may fail if the stream contains null characters.

301+-- TBD: Use UTF16LE on Windows.

302+fromChars :: MonadThrow m => Stream Identity Char -> m Path

303+fromChars s =

304+ let n = runIdentity $ Stream.fold Fold.length s

305+#if defined(mingw32_HOST_OS) || defined(__MINGW32__)

306+ in pure $ Path (Array.fromPureStreamN n (Unicode.encodeUtf16le' s))

307+#else

308+ in pure $ Path (Array.fromPureStreamN n (Unicode.encodeUtf8' s))

309+#endif

310+

311+-- | Decode the path to a stream of Unicode chars using strict UTF-8 decoding

312+-- on Posix.

313+-- TBD: Use UTF16LE on Windows.

314+toChars :: Monad m => Path -> Stream m Char

315+toChars (Path arr) =

316+#if defined(mingw32_HOST_OS) || defined(__MINGW32__)

317+ Unicode.decodeUtf16le' $ Array.read arr

318+#else

319+ Unicode.decodeUtf8' $ Array.read arr

320+#endif

321+

322+-- | Encode a Unicode string to 'Path' using strict UTF-8 encoding on Posix.

323+-- On Posix it may fail if the stream contains null characters.

324+-- TBD: Use UTF16LE on Windows.

325+fromString :: MonadThrow m => [Char] -> m Path

326+fromString = fromChars . Stream.fromList

327+

328+-- | Decode the path to a Unicode string using strict UTF-8 decoding on Posix.

329+-- TBD: Use UTF16LE on Windows.

330+toString :: Path -> [Char]

331+toString = runIdentity . Stream.toList . toChars

332+

333+------------------------------------------------------------------------------

334+-- Statically Verified Literals

335+------------------------------------------------------------------------------

336+

337+-- XXX Build these on top of the str quasiquoter so that we get the

338+-- interpolation for free.

339+

340+-- | Generates a 'Path' type from an interpolated string literal.

341+--

342+-- /Unimplemented/

343+path :: QuasiQuoter

344+path = undefined

345+

346+-- | Generates an @Abs Path@ type from an interpolated string literal.

347+--

348+-- /Unimplemented/

349+abs :: QuasiQuoter

350+abs = undefined

351+

352+-- | Generates an @Rel Path@ type from an interpolated string literal.

353+--

354+-- /Unimplemented/

355+rel :: QuasiQuoter

356+rel = undefined

357+

358+-- | Generates an @Dir Path@ type from an interpolated string literal.

359+--

360+-- /Unimplemented/

361+dir :: QuasiQuoter

362+dir = undefined

363+

364+-- | Generates an @File Path@ type from an interpolated string literal.

365+--

366+-- /Unimplemented/

367+file :: QuasiQuoter

368+file = undefined

369+

370+-- | Generates an @Abs (Dir Path)@ type from an interpolated string literal.

371+--

372+-- /Unimplemented/

373+absdir :: QuasiQuoter

374+absdir = undefined

375+

376+-- | Generates an @Rel (Dir Path)@ type from an interpolated string literal.

377+--

378+-- /Unimplemented/

379+reldir :: QuasiQuoter

380+reldir = undefined

381+

382+-- | Generates an @Abs (File Path)@ type from an interpolated string literal.

383+--

384+-- /Unimplemented/

385+absfile :: QuasiQuoter

386+absfile = undefined

387+

388+-- | Generates an @Rel (File Path)@ type from an interpolated string literal.

389+--

390+-- /Unimplemented/

391+relfile :: QuasiQuoter

392+relfile = undefined

393+

394+------------------------------------------------------------------------------

395+-- Statically Verified Strings

396+------------------------------------------------------------------------------

397+

398+-- | Generates a 'Path' type.

399+--

400+-- /Unimplemented/

401+mkPath :: String -> Q Exp

402+mkPath = undefined

403+

404+-- | Generates an @Abs Path@ type.

405+--

406+-- /Unimplemented/

407+mkAbs :: String -> Q Exp

408+mkAbs = undefined

409+

410+-- | Generates an @Rel Path@ type.

411+--

412+-- /Unimplemented/

413+mkRel :: String -> Q Exp

414+mkRel = undefined

415+

416+-- | Generates an @Dir Path@ type.

417+--

418+-- /Unimplemented/

419+mkDir :: String -> Q Exp

420+mkDir = undefined

421+

422+-- | Generates an @File Path@ type.

423+--

424+-- /Unimplemented/

425+mkFile :: String -> Q Exp

426+mkFile = undefined

427+

428+-- | Generates an @Abs (Dir Path)@ type.

429+--

430+-- /Unimplemented/

431+mkAbsDir :: String -> Q Exp

432+mkAbsDir = undefined

433+

434+-- | Generates an @Rel (Dir Path)@ type.

435+--

436+-- /Unimplemented/

437+mkRelDir :: String -> Q Exp

438+mkRelDir = undefined

439+

440+-- | Generates an @Abs (File Path)@ type.

441+--

442+-- /Unimplemented/

443+mkAbsFile :: String -> Q Exp

444+mkAbsFile = undefined

445+

446+-- | Generates an @Rel (File Path)@ type.

447+--

448+-- /Unimplemented/

449+mkRelFile :: String -> Q Exp

450+mkRelFile = undefined

451+

452+------------------------------------------------------------------------------

453+-- Operations

454+------------------------------------------------------------------------------

455+

456+separatorWord :: WORD_TYPE

457+separatorWord = SEPARATOR

458+

459+-- Portable definition for exporting.

460+

461+-- | Primary path separator character, @/@ on Posix and @\\@ on Windows.

462+-- Windows supports @/@ too as a separator. Please use 'isSeparator' for

463+-- testing if a char is a separator char.

464+primarySeparator :: Char

465+primarySeparator = chr (SEPARATOR)

466+

467+-- | On Posix only @/@ is a path separator but in windows it could be either

468+-- @/@ or @\\@.

469+isSeparator :: Char -> Bool

470+#if defined(mingw32_HOST_OS) || defined(__MINGW32__)

471+isSeparator c = (c == '/') || (c == '\\')

472+#else

473+isSeparator = (== '/')

474+#endif

475+

476+-- If we append an absolute path it may fail with an error if the 'Path'

477+-- implementation stores absolute path information (a leading separator char).

478+-- However, the implementation may choose to store the path as a list of

479+-- components in which case we cannot distinguish an absolute path from

480+-- relative.

481+

482+-- | Like 'extendDir' but for the less restrictive 'Path' type which will always

483+-- create a syntactically valid 'Path' type but it may not be semantically valid

484+-- because we may append an absolute path or we may append to a file path.

485+-- The onus lies on the user to ensure that the first path is not a file and

486+-- the second path is not absolute.

487+extendPath :: Path -> Path -> Path

488+extendPath (Path a) (Path b) =

489+ let len = Array.byteLength a + 1 + Array.byteLength b

490+ -- XXX Check the leading separator or drive identifier. However,

491+ -- checking the drive letter may add an additional overhead (can it be

492+ -- arbitrarily long?), if it is significant we may want to have a

493+ -- separate combinePathChecked API for that.

494+ --

495+ -- Also, do not add a separator char if the first path has a trailing

496+ -- separator.

497+ newArr = unsafePerformIO $ do

498+ arr <- MutArray.new len

499+ arr1 <- MutArray.spliceUnsafe arr (Array.unsafeThaw a)

500+ arr2 <- MutArray.snocUnsafe arr1 separatorWord

501+ arr3 <- MutArray.spliceUnsafe arr2 (Array.unsafeThaw b)

502+ return (Array.unsafeFreeze arr3)

503+ in Path newArr

504+

505+-- The only safety we need for paths is: (1) The first path can only be a Dir

506+-- type path, and (2) second path can only be a Rel path.

507+-- Can this be coerced to create unsafe versions?

508+

509+-- | Extend a directory path by appending a relative path to it. This is the

510+-- equivalent to the @</>@ operator from the @filepath@ package.

511+{-# INLINE extendDir #-}

512+extendDir :: (IsPath (a (Dir Path)), IsPath b, IsPath (a b)) =>

513+ (a (Dir Path)) -> Rel b -> a b

514+extendDir a (Rel b) =

515+ fromPathUnsafe $ extendPath (toPath a) (toPath b)

core/src/Streamly/Internal/Unicode/Parser.hs+3

@@ -300,6 +300,9 @@ data ScientificParseState

300300 | SPExponentWithSign !Multiplier !Number !DecimalPlaces !PowerMultiplier

301301 | SPAfterExponent !Multiplier !Number !DecimalPlaces !PowerMultiplier !Power

302302

303+-- XXX See https://hackage.haskell.org/package/integer-conversion for large

304+-- integers.

305+

303306-- | A generic parser for scientific notation of numbers. Returns (mantissa,

304307-- exponent) tuple. The result can be mapped to 'Double' or any other number

305308-- representation e.g. @Scientific@.

core/src/Streamly/Internal/Unicode/Stream.hs+36−9

@@ -28,6 +28,7 @@ module Streamly.Internal.Unicode.Stream

2828 , decodeUtf8

2929 , decodeUtf8'

3030 , decodeUtf8_

31+ , decodeUtf16le'

3132

3233 -- ** Resumable UTF-8 Decoding

3334 , DecodeError(..)

@@ -55,6 +56,7 @@ module Streamly.Internal.Unicode.Stream

5556 , encodeUtf8'

5657 , encodeUtf8_

5758 , encodeStrings

59+ , encodeUtf16le'

5860 {-

5961 -- * Operations on character strings

6062 , strip -- (dropAround isSpace)

@@ -97,7 +99,7 @@ import Data.Char (chr, ord)

9799#if MIN_VERSION_base(4,17,0)

98100import Data.Char (generalCategory, GeneralCategory(Space))

99101#endif

100−import Data.Word (Word8)

102+import Data.Word (Word8, Word16)

101103import Foreign.Marshal.Alloc (mallocBytes)

102104import Foreign.Storable (Storable(..))

103105#ifndef __GHCJS__

@@ -109,14 +111,14 @@ import GHC.IO.Encoding.Failure (isSurrogate)

109111import GHC.Ptr (Ptr (..), plusPtr)

110112import System.IO.Unsafe (unsafePerformIO)

111113import Streamly.Internal.Data.Array.Type (Array(..))

112−import Streamly.Internal.Data.MutArray.Type (MutableByteArray)

114+import Streamly.Internal.Data.MutByteArray.Type (MutByteArray)

113115import Streamly.Internal.Data.Fold (Fold)

114116import Streamly.Internal.Data.Parser (Parser)

115117import Streamly.Internal.Data.Stream (Stream)

116118import Streamly.Internal.Data.Stream (Step (..))

117119import Streamly.Internal.Data.SVar.Type (adaptState)

118120import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))

119−import Streamly.Internal.Data.Unbox (Unbox(peekByteIndex))

121+import Streamly.Internal.Data.Unbox (Unbox(peekAt))

120122import Streamly.Internal.Data.Unfold.Type (Unfold(..))

121123import Streamly.Internal.System.IO (unsafeInlineIO)

122124

@@ -676,6 +678,19 @@ decodeUtf8_ = decodeUtf8D_

676678decodeUtf8Lax :: Monad m => Stream m Word8 -> Stream m Char

677679decodeUtf8Lax = decodeUtf8

678680

681+-------------------------------------------------------------------------------

682+-- Decoding Utf16

683+-------------------------------------------------------------------------------

684+

685+-- | Decode a UTF-16 little endian encoded bytestream to a stream of Unicode

686+-- characters. The function throws an error if an invalid codepoint is

687+-- encountered.

688+--

689+-- /Unimplemented/

690+{-# INLINE decodeUtf16le' #-}

691+decodeUtf16le' :: Stream m Word16 -> Stream m Char

692+decodeUtf16le' = undefined

693+

679694-------------------------------------------------------------------------------

680695-- Decoding Array Streams

681696-------------------------------------------------------------------------------

@@ -685,9 +700,9 @@ decodeUtf8Lax = decodeUtf8

685700#endif

686701data FlattenState s

687702 = OuterLoop s !(Maybe (DecodeState, CodePoint))

688− | InnerLoopDecodeInit s MutableByteArray !Int !Int

689− | InnerLoopDecodeFirst s MutableByteArray !Int !Int Word8

690− | InnerLoopDecoding s MutableByteArray !Int !Int

703+ | InnerLoopDecodeInit s MutByteArray !Int !Int

704+ | InnerLoopDecodeFirst s MutByteArray !Int !Int Word8

705+ | InnerLoopDecoding s MutByteArray !Int !Int

691706 !DecodeState !CodePoint

692707 | YAndC !Char (FlattenState s) -- These constructors can be

693708 -- encoded in the UTF8DecodeState

@@ -746,7 +761,7 @@ decodeUtf8ArraysWithD cfm (D.Stream step state) =

746761 | p == end = do

747762 return $ Skip $ OuterLoop st Nothing

748763 step' _ _ (InnerLoopDecodeInit st contents p end) = do

749− x <- liftIO $ peekByteIndex p contents

764+ x <- liftIO $ peekAt p contents

750765 -- Note: It is important to use a ">" instead of a "<=" test here for

751766 -- GHC to generate code layout for default branch prediction for the

752767 -- common case. This is fragile and might change with the compiler

@@ -780,7 +795,7 @@ decodeUtf8ArraysWithD cfm (D.Stream step state) =

780795 step' _ _ (InnerLoopDecoding st _ p end sv cp)

781796 | p == end = return $ Skip $ OuterLoop st (Just (sv, cp))

782797 step' table _ (InnerLoopDecoding st contents p end statePtr codepointPtr) = do

783− x <- liftIO $ peekByteIndex p contents

798+ x <- liftIO $ peekAt p contents

784799 let (Tuple' sv cp) = decode1 table statePtr codepointPtr x

785800 return $

786801 case sv of

@@ -946,6 +961,18 @@ encodeUtf8_ = encodeUtf8D_

946961encodeUtf8Lax :: Monad m => Stream m Char -> Stream m Word8

947962encodeUtf8Lax = encodeUtf8

948963

964+-------------------------------------------------------------------------------

965+-- Encoding to Utf16

966+-------------------------------------------------------------------------------

967+

968+-- | Encode a stream of Unicode characters to a UTF-16 little endian encoded

969+-- bytestream.

970+--

971+-- /Unimplemented/

972+{-# INLINE encodeUtf16le' #-}

973+encodeUtf16le' :: Stream m Char -> Stream m Word16

974+encodeUtf16le' = undefined

975+

949976-------------------------------------------------------------------------------

950977-- Decoding string literals

951978-------------------------------------------------------------------------------

@@ -1030,7 +1057,7 @@ stripHead = Stream.dropWhile isSpace

10301057-- >>> Stream.fold Fold.toList $ Unicode.lines Fold.toList (Stream.fromList "lines\nthis\nstring\n\n\n")

10311058-- ["lines","this","string","",""]

10321059--

1033−-- > lines = Stream.splitOnSuffix (== '\n')

1060+-- >>> lines f = Stream.foldMany (Fold.takeEndBy_ (== '\n') f)

10341061--

10351062-- /Pre-release/

10361063{-# INLINE lines #-}

core/src/Streamly/Internal/Unicode/String.hs+7−7

@@ -39,14 +39,14 @@ module Streamly.Internal.Unicode.String

3939 --

4040 -- $setup

4141

42− str

42+ str

4343 ) where

4444

4545

4646import Control.Applicative (Alternative(..))

4747import Control.Exception (displayException)

4848import Data.Functor.Identity (runIdentity)

49−import Streamly.Internal.Data.Parser (Parser)

49+import Streamly.Internal.Data.Parser (Parser, ParseError)

5050

5151import Language.Haskell.TH

5252import Language.Haskell.TH.Quote

@@ -106,9 +106,12 @@ strSegmentExp (StrVar name) = do

106106strExp :: [StrSegment] -> Q Exp

107107strExp xs = appE [| concat |] $ listE $ map strSegmentExp xs

108108

109+parseStr :: String -> Either ParseError [StrSegment]

110+parseStr = runIdentity . Stream.parse strParser . Stream.fromList

111+

109112expandVars :: String -> Q Exp

110−expandVars ln =

111− case runIdentity $ Stream.parse strParser (Stream.fromList ln) of

113+expandVars input =

114+ case parseStr input of

112115 Left e ->

113116 fail $ "str QuasiQuoter parse error: " ++ displayException e

114117 Right x ->

@@ -141,9 +144,6 @@ expandVars ln =

141144-- :}

142145-- "hello world!"

143146--

144−-- Bugs: because of a bug in parsers, a lone # at the end of input gets

145−-- removed.

146−--

147147str :: QuasiQuoter

148148str =

149149 QuasiQuoter

core/streamly-core.cabal+83−35

@@ -1,8 +1,13 @@

11cabal-version: 2.2

22name: streamly-core

3−version: 0.2.0

4−synopsis: Streaming, parsers, arrays and more

3+version: 0.2.2

4+synopsis: Streaming, parsers, arrays, serialization and more

55description:

6+ For upgrading to streamly-0.9.0+ please read the

7+ <https://github.com/composewell/streamly/blob/streamly-0.10.0/docs/User/Project/Upgrading-0.8-to-0.9.md Streamly-0.9.0 upgrade guide>.

8+ .

9+ Streamly is a standard library for Haskell that focuses on C-like

10+ performance, modular combinators, and streaming data flow model.

611 Streamly consists of two packages: "streamly-core" and "streamly".

712 <https://hackage.haskell.org/package/streamly-core streamly-core>

813 provides basic features, and depends only on GHC boot libraries (see

@@ -12,8 +17,41 @@ description:

1217 and networking. For documentation, visit the

1318 <https://streamly.composewell.com Streamly website>.

1419 .

15− This package provides streams, arrays, parsers, unicode text, file

16− IO, and console IO functionality.

20+ The streamly-core package provides the following functionality:

21+ .

22+ * Streams as composable producers of a sequence of values.

23+ * Streams provide all the functionality provided by Haskell lists but

24+ in an effectful, streaming fashion with better performance.

25+ * Streams provide ListT and logic programming functionality as well.

26+ * Folds as composable stream consumers that reduce the streams to a

27+ single value or reduce segments of streams to transform the stream.

28+ * Parsers as more powerful, composable stream consumers supporting

29+ standard parser combinators with backtracking but in a streaming

30+ fashion.

31+ * Arrays with streaming interfaces providing high performance,

32+ modularity, and concise interface as all streaming operations can be

33+ performed on arrays.

34+ * Arrays can be immutable or mutable, unboxed or boxed, pinned or

35+ unpinned.

36+ * Arrays generalize the functionality provided by @bytestring@ and

37+ @text@ packages.

38+ * Interoperability with @bytestring@ and @text@ is

39+ provided via separate packages.

40+ * Arrays and folds provide natural builder functionality so there are no

41+ separate builder modules.

42+ * High performance binary serialization with configurable JSON like features.

43+ * Streaming combinators for unicode text processing, providing

44+ functionality equivalent to the @text@ package.

45+ * String interpolation for convenient construction of strings.

46+ * Streaming console IO (stdin/stdout) operations.

47+ * Streaming file and directory IO operations.

48+ .

49+ This package covers some or all of the functionality covered

50+ by @streaming, pipes, conduit, list-t, logic-t, foldl, attoparsec,

51+ array, primitive, vector, vector-algorithms, binary, cereal, store,

52+ bytestring, text, stringsearch, interpolate@. Streamly provides a

53+ consistent, concise, modular and performant interface for all this

54+ functionality.

1755 .

1856 Note: The dependencies "heaps" and "monad-control" are included in

1957 the package solely for backward compatibility, and will be removed in

@@ -50,6 +88,7 @@ extra-source-files:

5088 src/DocTestDataMutArray.hs

5189 src/DocTestDataMutArrayGeneric.hs

5290 src/DocTestDataParser.hs

91+ src/DocTestDataParserK.hs

5392 src/DocTestDataStream.hs

5493 src/DocTestDataStreamK.hs

5594 src/DocTestDataUnfold.hs

@@ -76,6 +115,8 @@ extra-doc-files:

76115 Changelog.md

77116 docs/*.md

78117 docs/ApiChangelogs/0.1.0.txt

118+ docs/ApiChangelogs/0.1.0-0.2.0.txt

119+ docs/ApiChangelogs/0.2.0-0.2.2.txt

79120

80121source-repository head

81122 type: git

@@ -177,8 +218,6 @@ common compile-options

177218common default-extensions

178219 default-extensions:

179220 BangPatterns

180− CApiFFI

181− CPP

182221 ConstraintKinds

183222 DeriveDataTypeable

184223 DeriveGeneric

@@ -189,24 +228,29 @@ common default-extensions

189228 GeneralizedNewtypeDeriving

190229 InstanceSigs

191230 KindSignatures

192− LambdaCase

193− MagicHash

194231 MultiParamTypeClasses

195− PatternSynonyms

196232 RankNTypes

197− RecordWildCards

198233 ScopedTypeVariables

199234 StandaloneDeriving

200235 TupleSections

201236 TypeApplications

202− TypeFamilies

203237 TypeOperators

204− ViewPatterns

205238

239+ -- Not GHC2021

240+ CApiFFI

241+ CPP

242+ DefaultSignatures

243+ LambdaCase

244+ MagicHash

245+ RecordWildCards

246+

247+ -- TypeFamilies is required by IsList, IsMap type classes and

248+ -- Unbox generic deriving code.

249+ -- TypeFamilies

206250 -- MonoLocalBinds, enabled by TypeFamilies, causes performance

207251 -- regressions. Disable it. This must come after TypeFamilies,

208252 -- otherwise TypeFamilies will enable it again.

209− NoMonoLocalBinds

253+ -- NoMonoLocalBinds

210254

211255 -- UndecidableInstances -- Does not show any perf impact

212256 -- UnboxedTuples -- interferes with (#.)

@@ -220,6 +264,7 @@ common optimization-options

220264

221265 -- For this to be effective it must come after the -O2 option

222266 if flag(dev) || flag(debug) || !flag(opt)

267+ cpp-options: -DDEBUG

223268 ghc-options: -fno-ignore-asserts

224269

225270common threading-options

@@ -291,16 +336,12 @@ library

291336 , Streamly.Internal.Data.Producer

292337

293338 -- streamly-core-array-types

294− , Streamly.Internal.Data.Unbox

295− , Streamly.Internal.Data.Serialize

296− , Streamly.Internal.Data.Unbox.TH

297− , Streamly.Internal.Data.Serialize.TH

298− , Streamly.Internal.Data.Serialize.TH.RecHeader

299− , Streamly.Internal.Data.Serialize.TH.Common

300− , Streamly.Internal.Data.Serialize.TH.Bottom

301−

302− -- Unboxed IORef

303− , Streamly.Internal.Data.IORef.Unboxed

339+ , Streamly.Internal.Data.MutByteArray

340+

341+ -- streaming and parsing Haskell types to/from bytes

342+ , Streamly.Internal.Data.Binary.Parser

343+ , Streamly.Internal.Data.Binary.Stream

344+

304345 -- May depend on streamly-core-stream

305346 , Streamly.Internal.Data.MutArray

306347 , Streamly.Internal.Data.MutArray.Generic

@@ -309,13 +350,11 @@ library

309350 , Streamly.Internal.Data.StreamK

310351 -- StreamD depends on streamly-array-types

311352 , Streamly.Internal.Data.Stream.StreamD

312− , Streamly.Internal.Data.Stream.Common

313353 , Streamly.Internal.Data.Stream

314354

315355 -- streamly-core-data

316356 , Streamly.Internal.Data.Builder

317357 , Streamly.Internal.Data.Unfold

318− , Streamly.Internal.Data.Fold.Tee

319358 , Streamly.Internal.Data.Fold.Chunked

320359 , Streamly.Internal.Data.Parser

321360 , Streamly.Internal.Data.ParserK

@@ -323,16 +362,12 @@ library

323362

324363 -- streamly-containers (non-base)

325364 , Streamly.Internal.Data.Fold

326− , Streamly.Internal.Data.Stream.Chunked

327365

328366 -- streamly-core-data-arrays

329367 , Streamly.Internal.Data.Array.Generic

330368 , Streamly.Internal.Data.Array

331− , Streamly.Internal.Data.Stream.MutChunked

332−

333− -- streamly-serde

334− , Streamly.Internal.Serialize.FromBytes

335− , Streamly.Internal.Serialize.ToBytes

369+ , Streamly.Internal.Data.MutArray.Stream

370+ , Streamly.Internal.Data.Array.Stream

336371

337372 -- streamly-unicode-core

338373 , Streamly.Internal.Unicode.Stream

@@ -341,6 +376,7 @@ library

341376 , Streamly.Internal.Unicode.Array

342377

343378 -- Filesystem/IO

379+ , Streamly.Internal.FileSystem.Path

344380 , Streamly.Internal.FileSystem.Handle

345381 , Streamly.Internal.FileSystem.File

346382 , Streamly.Internal.FileSystem.Dir

@@ -378,7 +414,7 @@ library

378414 -- streamly-core released modules in alphabetic order

379415 -- NOTE: these must be added to streamly.cabal as well

380416 , Streamly.Console.Stdio

381− , Streamly.Data.Serialize

417+ , Streamly.Data.MutByteArray

382418 , Streamly.Data.Array

383419 , Streamly.Data.Array.Generic

384420 , Streamly.Data.MutArray

@@ -400,6 +436,7 @@ library

400436 , Streamly.Internal.Data.Fold.Type

401437 , Streamly.Internal.Data.Fold.Combinators

402438 , Streamly.Internal.Data.Fold.Container

439+ , Streamly.Internal.Data.Fold.Tee

403440 , Streamly.Internal.Data.Fold.Window

404441

405442 , Streamly.Internal.Data.Parser.Type

@@ -430,7 +467,17 @@ library

430467

431468 , Streamly.Internal.Data.Array.Type

432469

470+ -- Unboxed IORef

471+ , Streamly.Internal.Data.IORef.Unboxed

472+

473+ , Streamly.Internal.Data.MutByteArray.Type

474+ , Streamly.Internal.Data.Unbox

475+ , Streamly.Internal.Data.Unbox.TH

433476 , Streamly.Internal.Data.Serialize.Type

477+ , Streamly.Internal.Data.Serialize.TH

478+ , Streamly.Internal.Data.Serialize.TH.RecHeader

479+ , Streamly.Internal.Data.Serialize.TH.Common

480+ , Streamly.Internal.Data.Serialize.TH.Bottom

434481

435482 , Streamly.Internal.Data.Producer.Type

436483 , Streamly.Internal.Data.Producer.Source

@@ -438,9 +485,10 @@ library

438485 , Streamly.Internal.Data.Time.Clock.Type

439486

440487 if flag(dev)

441− other-modules:

488+ exposed-modules:

442489 Streamly.Internal.Data.StreamK.Alt

443− , Streamly.Internal.Data.List

490+ -- XXX Compilation needs to be fixed

491+ -- , Streamly.Internal.Data.List

444492

445493 build-depends:

446494 -- streamly-base

@@ -459,7 +507,7 @@ library

459507 , base >= 4.12 && < 4.20

460508 , exceptions >= 0.8.0 && < 0.11

461509 , transformers >= 0.5.5 && < 0.7

462− , filepath >= 1.4.2 && < 1.5

510+ , filepath >= 1.4.2 && < 1.6

463511

464512 -- streamly-unicode-core

465513 , template-haskell >= 2.14 && < 2.22

docs/Developer/optimization-guidelines.md+12

@@ -191,6 +191,18 @@ the stages of a loop, it is often not useful to inline the whole loop

191191itself, in fact we may have to occasionally use a `NOINLINE` so that the

192192compiler does not inline it.

193193

194+We need to be careful about NOINLINE on polymorphic functions. When

195+NOINLINE is used, the function cannot be specialized causing huge

196+performance degradation. To avoid that we can either make the

197+function monomorphic or sometimes using `NOINLINE [0]` can do the

198+trick. `NOINLINE [0]` blocks inlining in earlier phases but lets it

199+specialize in the last phase, inlining in the last phase is left to

200+the compiler. `INLINE [0]` may sound the same as `NOINLINE [0]`,

201+however, it behaves differently because we ask the compiler to INLINE

202+it compulsorily and it may not give us the desired results. The

203+Data.Fold.writeLastN benchmark is one such case where `NOINLINE [0]`

204+provides much better performance than `INLINE [0]`.

205+

194206### NoSpecConstr

195207

196208It is not always useful to specialize function calls for all constructors, some

docs/Developer/paths.rst+25−2

@@ -140,12 +140,35 @@ know exactly how the file system stores the path. Otherwise if we display

140140it differently, the user may use the displayed result to find the file

141141and may not find it.

142142

143+OS Specific Path Encodings

144+--------------------------

145+

146+In general, the OS does not have to look inside the path components

147+other than the ability to recognize the separator char to parse the

148+path into components and to construct it from components. However, in

149+some cases the OS may may have to look inside the path e.g. "." or ".."

150+special paths on Unix.

151+

152+On Unix a path component is just a blob of bytes. The user can encode

153+and decode this blob of bytes as they desire. All they need to care

154+about is that the path separator byte is respected. However, the the

155+path component blobs may be changed by the underlying filesystem

156+e.g. apple HFS may normalize the path before storing, therefore, the

157+representation may change when it is returned back to the user.

158+

159+On modern Windows systems a path component is UTF16-LE

160+encoded. Theoretically, from the OS perspective, it does not have to

161+look inside the path components other than determining whether a 16-bit

162+unit represents the separator character, therefore, users can use any

163+encoding as long as it encodes to 16-bit multiples and the separator is

164+preserved as equivalent to the UTF16-LE representation.

165+

143166Type Safety Requirements

144167------------------------

145168

146169* Safety against using an absolute path where a relative path is to be

147− used and vice-versa.

148−

170+ used and vice-versa.

171+

149172 * Validations for absolute or relative path when constructing a path.

150173 * We cannot append an absolute path to another path

151174* Safety against using a file name where a directory name is to be used and

docs/User/Project/ApiChangelogs/0.9.0-0.10.0.txtnew file+424

@@ -0,0 +1,424 @@

1+---------------------------------

2+API Annotations

3+---------------------------------

5+[A] : Added

6+[R] : Removed

7+[C] : Changed

8+[O] : Old definition

9+[N] : New definition

10+[D] : Deprecated

11+

12+---------------------------------

13+API diff

14+---------------------------------

15+

16+[C] Streamly.Network.Socket

17+ [A] readWith :: MonadIO m => Int -> Socket -> Stream m Word8

18+ [A] readChunksWith :: MonadIO m => Int -> Socket -> Stream m (Array Word8)

19+[C] Streamly.Network.Inet.TCP

20+ [D] acceptorOnPortLocal :: MonadIO m => Unfold m PortNumber Socket

21+ [D] acceptorOnPort :: MonadIO m => Unfold m PortNumber Socket

22+ [A] acceptorLocal :: MonadIO m => Unfold m PortNumber Socket

23+ [A] acceptor :: MonadIO m => Unfold m PortNumber Socket

24+ [A] acceptOnAddrWith :: MonadIO m => [(SocketOption, Int)] -> (Word8, Word8, Word8, Word8) -> PortNumber -> Stream m Socket

25+ [A] acceptLocal :: MonadIO m => PortNumber -> Stream m Socket

26+ [A] accept :: MonadIO m => PortNumber -> Stream m Socket

27+[C] Streamly.Data.Stream.Prelude

28+ [D] tapCount :: MonadAsync m => (a -> Bool) -> (Stream m Int -> m b) -> Stream m a -> Stream m a

29+ [A] parTapCount :: MonadAsync m => (a -> Bool) -> (Stream m Int -> m b) -> Stream m a -> Stream m a

30+[A] Streamly.Data.Fold.Prelude

31+ [A] Config

32+ [A] toHashMapIO :: (MonadIO m, Hashable k, Ord k) => (a -> k) -> Fold m a b -> Fold m a (HashMap k b)

33+ [A] parEval :: MonadAsync m => (Config -> Config) -> Fold m a b -> Fold m a b

34+ [A] maxBuffer :: Int -> Config -> Config

35+ [A] inspect :: Bool -> Config -> Config

36+ [A] boundThreads :: Bool -> Config -> Config

37+

38+---------------------------------

39+Internal API diff

40+---------------------------------

41+

42+[C] Streamly.Internal.Network.Inet.TCP

43+ [A] acceptorWith :: MonadIO m => [(SocketOption, Int)] -> Unfold m PortNumber Socket

44+ [R] acceptorOnPortWith :: MonadIO m => [(SocketOption, Int)] -> Unfold m PortNumber Socket

45+ [D] acceptorOnPortLocal :: MonadIO m => Unfold m PortNumber Socket

46+ [D] acceptorOnPort :: MonadIO m => Unfold m PortNumber Socket

47+ [A] acceptorLocal :: MonadIO m => Unfold m PortNumber Socket

48+ [A] acceptor :: MonadIO m => Unfold m PortNumber Socket

49+ [R] acceptOnPortLocal :: MonadIO m => PortNumber -> Stream m Socket

50+ [R] acceptOnPort :: MonadIO m => PortNumber -> Stream m Socket

51+ [A] acceptLocal :: MonadIO m => PortNumber -> Stream m Socket

52+ [A] accept :: MonadIO m => PortNumber -> Stream m Socket

53+[R] Streamly.Internal.Data.Unfold.SVar

54+[A] Streamly.Internal.Data.Unfold.Prelude

55+ [A] gbracket :: MonadRunInIO m => (a -> m c) -> (c -> m d) -> Unfold m (c, e) b -> (forall s. m s -> m (Either e s)) -> Unfold m c b -> Unfold m a b

56+ [A] fromSVar :: MonadAsync m => Unfold m (SVar t m a) a

57+ [A] fromProducer :: MonadAsync m => Unfold m (SVar t m a) a

58+ [A] finally :: (MonadAsync m, MonadCatch m) => (a -> m c) -> Unfold m a b -> Unfold m a b

59+ [A] bracket :: (MonadAsync m, MonadCatch m) => (a -> m c) -> (c -> m d) -> Unfold m c b -> Unfold m a b

60+ [A] after :: MonadRunInIO m => (a -> m c) -> Unfold m a b -> Unfold m a b

61+[R] Streamly.Internal.Data.Unfold.Exception

62+[R] Streamly.Internal.Data.Stream.Zip.Concurrent

63+[D] Streamly.Internal.Data.Stream.Zip

64+ [C] GHC.Base.Functor

65+ [A] instance GHC.Base.Monad m => GHC.Base.Functor (Streamly.Internal.Data.Stream.Zip.ZipConcurrent m)

66+ [C] GHC.Base.Applicative

67+ [A] instance Streamly.Internal.Control.Concurrent.MonadAsync m => GHC.Base.Applicative (Streamly.Internal.Data.Stream.Zip.ZipConcurrent m)

68+ [A] ZipConcurrent

69+ [A] [getZipConcurrent] :: ZipConcurrent m a -> Stream m a

70+ [A] ZipConcurrent :: Stream m a -> ZipConcurrent m a

71+[R] Streamly.Internal.Data.Stream.Time

72+[R] Streamly.Internal.Data.Stream.SVar.Generate

73+[R] Streamly.Internal.Data.Stream.SVar.Eliminate

74+[A] Streamly.Internal.Data.Stream.Prelude

75+ [A] StopWhen

76+ [A] FirstStops :: StopWhen

77+ [A] AnyStops :: StopWhen

78+ [A] AllStop :: StopWhen

79+ [A] Rate

80+ [A] [rateLow] :: Rate -> Double

81+ [A] [rateHigh] :: Rate -> Double

82+ [A] [rateGoal] :: Rate -> Double

83+ [A] [rateBuffer] :: Rate -> Int

84+ [A] Rate :: Double -> Double -> Double -> Int -> Rate

85+ [A] Config

86+ [A] Channel

87+ [A] [yieldRateInfo] :: Channel m a -> Maybe YieldRateInfo

88+ [A] [workerThreads] :: Channel m a -> IORef (Set ThreadId)

89+ [A] [workerStopMVar] :: Channel m a -> MVar ()

90+ [A] [workerCount] :: Channel m a -> IORef Int

91+ [A] [workLoop] :: Channel m a -> Maybe WorkerInfo -> m ()

92+ [A] [svarStats] :: Channel m a -> SVarStats

93+ [A] [svarRef] :: Channel m a -> Maybe (IORef ())

94+ [A] [svarMrun] :: Channel m a -> RunInIO m

95+ [A] [svarInspectMode] :: Channel m a -> Bool

96+ [A] [svarCreator] :: Channel m a -> ThreadId

97+ [A] [remainingWork] :: Channel m a -> Maybe (IORef Count)

98+ [A] [readOutputQ] :: Channel m a -> m [ChildEvent a]

99+ [A] [postProcess] :: Channel m a -> m Bool

100+ [A] [outputQueue] :: Channel m a -> IORef ([ChildEvent a], Int)

101+ [A] [outputDoorBell] :: Channel m a -> MVar ()

102+ [A] [maxWorkerLimit] :: Channel m a -> Limit

103+ [A] [maxBufferLimit] :: Channel m a -> Limit

104+ [A] [isWorkDone] :: Channel m a -> IO Bool

105+ [A] [isQueueDone] :: Channel m a -> IO Bool

106+ [A] [enqueue] :: Channel m a -> Bool -> (RunInIO m, StreamK m a) -> IO ()

107+ [A] [eagerDispatch] :: Channel m a -> m ()

108+ [A] [doorBellOnWorkQ] :: Channel m a -> IORef Bool

109+ [A] [accountThread] :: Channel m a -> ThreadId -> m ()

110+ [A] Channel :: RunInIO m -> IORef ([ChildEvent a], Int) -> MVar () -> m [ChildEvent a] -> m Bool -> Limit -> Limit -> Maybe (IORef Count) -> Maybe YieldRateInfo -> (Bool -> (RunInIO m, StreamK m a) -> IO ()) -> m () -> IO Bool -> IO Bool -> IORef Bool -> (Maybe WorkerInfo -> m ()) -> IORef (Set ThreadId) -> IORef Int -> (ThreadId -> m ()) -> MVar () -> Maybe (IORef ()) -> SVarStats -> Bool -> ThreadId -> Channel m a

111+ [A] type MonadAsync m = (MonadIO m, MonadBaseControl IO m, MonadThrow m)

112+ [A] yield :: Channel m a -> Maybe WorkerInfo -> a -> IO Bool

113+ [A] withChannelK :: MonadAsync m => (Config -> Config) -> StreamK m a -> (Channel m b -> StreamK m a -> StreamK m b) -> StreamK m b

114+ [A] withChannel :: MonadAsync m => (Config -> Config) -> Stream m a -> (Channel m b -> Stream m a -> Stream m b) -> Stream m b

115+ [A] toChannelK :: MonadRunInIO m => Channel m a -> StreamK m a -> m ()

116+ [A] toChannel :: MonadRunInIO m => Channel m a -> Stream m a -> m ()

117+ [A] ticksRate :: MonadAsync m => Rate -> Stream m ()

118+ [A] ticks :: MonadIO m => Double -> Stream m ()

119+ [A] takeLastInterval :: Double -> Stream m a -> Stream m a

120+ [A] takeInterval :: MonadAsync m => Double -> Stream m a -> Stream m a

121+ [A] stopWhen :: StopWhen -> Config -> Config

122+ [A] stopChannel :: MonadIO m => Channel m a -> m ()

123+ [A] stop :: Channel m a -> Maybe WorkerInfo -> IO ()

124+ [A] startChannel :: MonadRunInIO m => Channel m a -> m ()

125+ [A] sendWorkerWait :: MonadIO m => Bool -> (Channel m a -> IO ()) -> (Channel m a -> m Bool) -> Channel m a -> m ()

126+ [A] sendWorkerDelayPaced :: Channel m a -> IO ()

127+ [A] sendWorkerDelay :: Channel m a -> IO ()

128+ [A] sampleIntervalStart :: MonadAsync m => Double -> Stream m a -> Stream m a

129+ [A] sampleIntervalEnd :: MonadAsync m => Double -> Stream m a -> Stream m a

130+ [A] sampleBurstStart :: MonadAsync m => Double -> Stream m a -> Stream m a

131+ [A] sampleBurstEnd :: MonadAsync m => Double -> Stream m a -> Stream m a

132+ [A] sampleBurst :: MonadAsync m => Bool -> Double -> Stream m a -> Stream m a

133+ [A] retryD :: forall e m a. (Exception e, Ord e, MonadCatch m) => Map e Int -> (e -> Stream m a) -> Stream m a -> Stream m a

134+ [A] retry :: (MonadCatch m, Exception e, Ord e) => Map e Int -> (e -> Stream m a) -> Stream m a -> Stream m a

135+ [A] readOutputQPaced :: MonadRunInIO m => Channel m a -> m [ChildEvent a]

136+ [A] readOutputQBounded :: MonadRunInIO m => Bool -> Channel m a -> m [ChildEvent a]

137+ [A] rate :: Maybe Rate -> Config -> Config

138+ [A] pushWorker :: MonadRunInIO m => Count -> Channel m a -> m ()

139+ [A] postProcessPaced :: MonadRunInIO m => Channel m a -> m Bool

140+ [A] postProcessBounded :: MonadRunInIO m => Channel m a -> m Bool

141+ [A] periodic :: MonadIO m => m a -> Double -> Stream m a

142+ [A] parZipWithM :: MonadAsync m => (Config -> Config) -> (a -> b -> m c) -> Stream m a -> Stream m b -> Stream m c

143+ [A] parZipWith :: MonadAsync m => (Config -> Config) -> (a -> b -> c) -> Stream m a -> Stream m b -> Stream m c

144+ [A] parTwo :: MonadAsync m => (Config -> Config) -> Stream m a -> Stream m a -> Stream m a

145+ [A] parTapCount :: MonadAsync m => (a -> Bool) -> (Stream m Int -> m b) -> Stream m a -> Stream m a

146+ [A] parSequence :: MonadAsync m => (Config -> Config) -> Stream m (m a) -> Stream m a

147+ [A] parReplicateM :: MonadAsync m => (Config -> Config) -> Int -> m a -> Stream m a

148+ [A] parRepeatM :: MonadAsync m => (Config -> Config) -> m a -> Stream m a

149+ [A] parMergeByM :: MonadAsync m => (Config -> Config) -> (a -> a -> m Ordering) -> Stream m a -> Stream m a -> Stream m a

150+ [A] parMergeBy :: MonadAsync m => (Config -> Config) -> (a -> a -> Ordering) -> Stream m a -> Stream m a -> Stream m a

151+ [A] parMapM :: MonadAsync m => (Config -> Config) -> (a -> m b) -> Stream m a -> Stream m b

152+ [A] parListOrdered :: MonadAsync m => [Stream m a] -> Stream m a

153+ [A] parListLazy :: MonadAsync m => [Stream m a] -> Stream m a

154+ [A] parListInterleaved :: MonadAsync m => [Stream m a] -> Stream m a

155+ [A] parListEagerMin :: MonadAsync m => [Stream m a] -> Stream m a

156+ [A] parListEagerFst :: MonadAsync m => [Stream m a] -> Stream m a

157+ [A] parListEager :: MonadAsync m => [Stream m a] -> Stream m a

158+ [A] parList :: MonadAsync m => (Config -> Config) -> [Stream m a] -> Stream m a

159+ [A] parEval :: MonadAsync m => (Config -> Config) -> Stream m a -> Stream m a

160+ [A] parConcatMap :: MonadAsync m => (Config -> Config) -> (a -> Stream m b) -> Stream m a -> Stream m b

161+ [A] parConcatIterate :: MonadAsync m => (Config -> Config) -> (a -> Stream m a) -> Stream m a -> Stream m a

162+ [A] parConcat :: MonadAsync m => (Config -> Config) -> Stream m (Stream m a) -> Stream m a

163+ [A] parApply :: MonadAsync m => (Config -> Config) -> Stream m (a -> b) -> Stream m a -> Stream m b

164+ [A] ordered :: Bool -> Config -> Config

165+ [A] newInterleaveChannel :: MonadAsync m => (Config -> Config) -> m (Channel m a)

166+ [A] newChannel :: MonadAsync m => (Config -> Config) -> m (Channel m a)

167+ [A] newAppendChannel :: MonadRunInIO m => (Config -> Config) -> m (Channel m a)

168+ [A] minRate :: Double -> Config -> Config

169+ [A] maxThreads :: Int -> Config -> Config

170+ [A] maxRate :: Double -> Config -> Config

171+ [A] maxBuffer :: Int -> Config -> Config

172+ [A] intervalsOf :: MonadAsync m => Double -> Fold m a b -> Stream m a -> Stream m b

173+ [A] interleaved :: Bool -> Config -> Config

174+ [A] interject :: MonadAsync m => m a -> Double -> Stream m a -> Stream m a

175+ [A] inspect :: Bool -> Config -> Config

176+ [A] groupsOfTimeout :: MonadAsync m => Int -> Double -> Fold m a b -> Stream m a -> Stream m b

177+ [A] getStopWhen :: Config -> StopWhen

178+ [A] fromChannelK :: MonadAsync m => Channel m a -> StreamK m a

179+ [A] fromChannel :: MonadAsync m => Channel m a -> Stream m a

180+ [A] fromCallback :: MonadAsync m => ((a -> m ()) -> m ()) -> Stream m a

181+ [A] finally :: (MonadAsync m, MonadCatch m) => m b -> Stream m a -> Stream m a

182+ [A] eager :: Bool -> Config -> Config

183+ [A] dumpSVar :: Channel m a -> IO String

184+ [A] dropLastInterval :: Int -> Stream m a -> Stream m a

185+ [A] dropInterval :: MonadAsync m => Double -> Stream m a -> Stream m a

186+ [A] dispatchWorkerPaced :: MonadRunInIO m => Channel m a -> m Bool

187+ [A] dispatchWorker :: MonadRunInIO m => Count -> Channel m a -> m Bool

188+ [A] defaultConfig :: Config

189+ [A] constRate :: Double -> Config -> Config

190+ [A] classifySessionsOf :: (MonadAsync m, Ord k) => (Int -> m Bool) -> Double -> Fold m a b -> Stream m (AbsTime, (k, a)) -> Stream m (k, b)

191+ [A] classifySessionsByGeneric :: forall m f a b. (MonadAsync m, IsMap f) => Proxy (f :: Type -> Type) -> Double -> Bool -> (Int -> m Bool) -> Double -> Fold m a b -> Stream m (AbsTime, (Key f, a)) -> Stream m (Key f, b)

192+ [A] classifySessionsBy :: (MonadAsync m, Ord k) => Double -> Bool -> (Int -> m Bool) -> Double -> Fold m a b -> Stream m (AbsTime, (k, a)) -> Stream m (k, b)

193+ [A] classifyKeepAliveSessions :: (MonadAsync m, Ord k) => (Int -> m Bool) -> Double -> Fold m a b -> Stream m (AbsTime, (k, a)) -> Stream m (k, b)

194+ [A] bufferOldestN :: Int -> Stream m a -> Stream m a

195+ [A] bufferLatestN :: Int -> Stream m a -> Stream m a

196+ [A] bufferLatest :: Stream m a -> Stream m (Maybe a)

197+ [A] bracket3D :: (MonadAsync m, MonadCatch m) => m b -> (b -> m c) -> (b -> m d) -> (b -> m e) -> (b -> Stream m a) -> Stream m a

198+ [A] bracket3 :: (MonadAsync m, MonadCatch m) => m b -> (b -> m c) -> (b -> m d) -> (b -> m e) -> (b -> Stream m a) -> Stream m a

199+ [A] bracket :: (MonadAsync m, MonadCatch m) => m b -> (b -> m c) -> (b -> Stream m a) -> Stream m a

200+ [A] avgRate :: Double -> Config -> Config

201+ [A] afterD :: MonadRunInIO m => m b -> Stream m a -> Stream m a

202+ [A] after :: (MonadIO m, MonadBaseControl IO m) => m b -> Stream m a -> Stream m a

203+[R] Streamly.Internal.Data.Stream.Exception.Lifted

204+[R] Streamly.Internal.Data.Stream.Concurrent.Channel.Type

205+[R] Streamly.Internal.Data.Stream.Concurrent.Channel.Operations

206+[R] Streamly.Internal.Data.Stream.Concurrent.Channel.Interleave

207+[R] Streamly.Internal.Data.Stream.Concurrent.Channel.Dispatcher

208+[R] Streamly.Internal.Data.Stream.Concurrent.Channel.Consumer

209+[R] Streamly.Internal.Data.Stream.Concurrent.Channel.Append

210+[R] Streamly.Internal.Data.Stream.Concurrent.Channel

211+[R] Streamly.Internal.Data.Stream.Concurrent

212+[R] Streamly.Internal.Data.Stream.Channel.Worker

213+[R] Streamly.Internal.Data.Stream.Channel.Types

214+[R] Streamly.Internal.Data.Stream.Channel.Dispatcher

215+[R] Streamly.Internal.Data.SVar.Worker

216+[R] Streamly.Internal.Data.SVar.Pull

217+[R] Streamly.Internal.Data.SVar.Dispatch

218+[D] Streamly.Internal.Data.SVar

219+ [A] Work

220+ [A] PartialWorker :: Count -> Work

221+ [A] ManyWorkers :: Int -> Count -> Work

222+ [A] BlockWait :: NanoSecond64 -> Work

223+ [A] workerUpdateLatency :: YieldRateInfo -> WorkerInfo -> IO ()

224+ [A] workerRateControl :: SVar t m a -> YieldRateInfo -> WorkerInfo -> IO Bool

225+ [A] withDiagMVar :: SVar t m a -> String -> IO () -> IO ()

226+ [A] updateYieldCount :: WorkerInfo -> IO Count

227+ [A] sendYield :: SVar t m a -> Maybe WorkerInfo -> ChildEvent a -> IO Bool

228+ [A] sendWorkerWait :: MonadAsync m => (SVar t m a -> IO ()) -> (SVar t m a -> m Bool) -> SVar t m a -> m ()

229+ [A] sendWorkerDelayPaced :: SVar t m a -> IO ()

230+ [A] sendWorkerDelay :: SVar t m a -> IO ()

231+ [A] sendToProducer :: SVar t m a -> ChildEvent a -> IO Int

232+ [A] sendStopToProducer :: MonadIO m => SVar t m a -> m ()

233+ [A] sendStop :: SVar t m a -> Maybe WorkerInfo -> IO ()

234+ [A] sendFirstWorker :: MonadAsync m => SVar t m a -> t m a -> m (SVar t m a)

235+ [A] send :: SVar t m a -> ChildEvent a -> IO Int

236+ [A] ringDoorBell :: SVar t m a -> IO ()

237+ [A] resetBufferLimit :: SVar t m a -> IO ()

238+ [A] recordMaxWorkers :: MonadIO m => SVar t m a -> m ()

239+ [A] readOutputQRaw :: SVar t m a -> IO ([ChildEvent a], Int)

240+ [A] readOutputQPaced :: MonadAsync m => SVar t m a -> m [ChildEvent a]

241+ [A] readOutputQBounded :: MonadAsync m => SVar t m a -> m [ChildEvent a]

242+ [A] readOutputQBasic :: IORef ([ChildEvent a], Int) -> IO ([ChildEvent a], Int)

243+ [A] pushWorkerPar :: MonadAsync m => SVar t m a -> (Maybe WorkerInfo -> m ()) -> m ()

244+ [A] pushWorker :: MonadAsync m => Count -> SVar t m a -> m ()

245+ [A] printSVar :: SVar t m a -> String -> IO ()

246+ [A] postProcessPaced :: MonadAsync m => SVar t m a -> m Bool

247+ [A] postProcessBounded :: MonadAsync m => SVar t m a -> m Bool

248+ [A] modifyThread :: MonadIO m => SVar t m a -> ThreadId -> m ()

249+ [A] minThreadDelay :: NanoSecond64

250+ [A] isBeyondMaxRate :: SVar t m a -> YieldRateInfo -> IO Bool

251+ [A] incrementYieldLimit :: SVar t m a -> IO ()

252+ [A] incrementBufferLimit :: SVar t m a -> IO ()

253+ [A] handleFoldException :: SVar t m a -> SomeException -> IO ()

254+ [A] handleChildException :: SVar t m a -> SomeException -> IO ()

255+ [A] estimateWorkers :: Limit -> Count -> Count -> NanoSecond64 -> NanoSecond64 -> NanoSecond64 -> LatencyRange -> Work

256+ [A] dumpSVar :: SVar t m a -> IO String

257+ [A] dispatchWorkerPaced :: MonadAsync m => SVar t m a -> m Bool

258+ [A] dispatchWorker :: MonadAsync m => Count -> SVar t m a -> m Bool

259+ [A] delThread :: MonadIO m => SVar t m a -> ThreadId -> m ()

260+ [A] decrementYieldLimit :: SVar t m a -> IO Bool

261+ [A] decrementBufferLimit :: SVar t m a -> IO ()

262+ [A] collectLatency :: SVar t m a -> YieldRateInfo -> Bool -> IO (Count, AbsTime, NanoSecond64)

263+ [A] cleanupSVarFromWorker :: SVar t m a -> IO ()

264+ [A] cleanupSVar :: SVar t m a -> IO ()

265+ [A] allThreadsDone :: MonadIO m => SVar t m a -> m Bool

266+[C] Streamly.Internal.Data.IsMap.HashMap

267+ [C] Streamly.Internal.Data.IsMap.IsMap

268+ [C] (Data.HashMap.Internal.HashMap k)

269+ [O] instance (Data.Hashable.Class.Hashable k, GHC.Classes.Eq k) => Streamly.Internal.Data.IsMap.IsMap (Data.HashMap.Internal.HashMap k)

270+ [N] instance Data.Hashable.Class.Hashable k => Streamly.Internal.Data.IsMap.IsMap (Data.HashMap.Internal.HashMap k)

271+[C] Streamly.Internal.Data.IOFinalizer.Lifted

272+ [C] IOFinalizer

273+[R] Streamly.Internal.Data.Fold.SVar

274+[A] Streamly.Internal.Data.Fold.Prelude

275+ [A] Config

276+ [A] Channel

277+ [A] [svarStats] :: Channel m a b -> SVarStats

278+ [A] [svarRef] :: Channel m a b -> Maybe (IORef ())

279+ [A] [svarInspectMode] :: Channel m a b -> Bool

280+ [A] [svarCreator] :: Channel m a b -> ThreadId

281+ [A] [readOutputQ] :: Channel m a b -> m [ChildEvent a]

282+ [A] [outputQueue] :: Channel m a b -> IORef ([ChildEvent a], Int)

283+ [A] [outputQueueFromConsumer] :: Channel m a b -> IORef ([ChildEvent b], Int)

284+ [A] [outputDoorBell] :: Channel m a b -> MVar ()

285+ [A] [outputDoorBellFromConsumer] :: Channel m a b -> MVar ()

286+ [A] [maxBufferLimit] :: Channel m a b -> Limit

287+ [A] [bufferSpaceDoorBell] :: Channel m a b -> MVar ()

288+ [A] Channel :: IORef ([ChildEvent a], Int) -> Limit -> MVar () -> m [ChildEvent a] -> IORef ([ChildEvent b], Int) -> MVar () -> MVar () -> Maybe (IORef ()) -> SVarStats -> Bool -> ThreadId -> Channel m a b

289+ [A] writeLimited :: MonadIO m => SVar t m a -> Maybe WorkerInfo -> Fold m a ()

290+ [A] write :: MonadIO m => SVar t m a -> Maybe WorkerInfo -> Fold m a ()

291+ [A] takeInterval :: MonadAsync m => Double -> Fold m a b -> Fold m a b

292+ [A] sendToWorker :: MonadAsync m => Channel m a b -> a -> m (Maybe b)

293+ [A] parEval :: MonadAsync m => (Config -> Config) -> Fold m a b -> Fold m a b

294+ [A] newChannel :: MonadRunInIO m => (Config -> Config) -> Fold m a b -> m (Channel m a b)

295+ [A] maxBuffer :: Int -> Config -> Config

296+ [A] intervalsOf :: MonadAsync m => Double -> Fold m a b -> Fold m b c -> Fold m a c

297+ [A] inspect :: Bool -> Config -> Config

298+ [A] dumpSVar :: Channel m a b -> IO String

299+ [A] checkFoldStatus :: MonadAsync m => Channel m a b -> m (Maybe b)

300+ [A] boundThreads :: Bool -> Config -> Config

301+[R] Streamly.Internal.Data.Fold.Concurrent.Channel.Type

302+[R] Streamly.Internal.Data.Fold.Concurrent.Channel

303+[R] Streamly.Internal.Data.Fold.Concurrent

304+[R] Streamly.Internal.Data.Fold.Async

305+[A] Streamly.Internal.Data.Channel

306+ [A] YieldRateInfo

307+ [A] [workerPollingInterval] :: YieldRateInfo -> IORef Count

308+ [A] [workerPendingLatency] :: YieldRateInfo -> IORef (Count, Count, NanoSecond64)

309+ [A] [workerMeasuredLatency] :: YieldRateInfo -> IORef NanoSecond64

310+ [A] [workerCollectedLatency] :: YieldRateInfo -> IORef (Count, Count, NanoSecond64)

311+ [A] [workerBootstrapLatency] :: YieldRateInfo -> Maybe NanoSecond64

312+ [A] [svarRateBuffer] :: YieldRateInfo -> Int

313+ [A] [svarLatencyTarget] :: YieldRateInfo -> NanoSecond64

314+ [A] [svarLatencyRange] :: YieldRateInfo -> LatencyRange

315+ [A] [svarGainedLostYields] :: YieldRateInfo -> IORef Count

316+ [A] [svarAllTimeLatency] :: YieldRateInfo -> IORef (Count, AbsTime)

317+ [A] YieldRateInfo :: NanoSecond64 -> LatencyRange -> Int -> IORef Count -> IORef (Count, AbsTime) -> Maybe NanoSecond64 -> IORef Count -> IORef (Count, Count, NanoSecond64) -> IORef (Count, Count, NanoSecond64) -> IORef NanoSecond64 -> YieldRateInfo

318+ [A] WorkerInfo

319+ [A] [workerYieldMax] :: WorkerInfo -> Count

320+ [A] [workerYieldCount] :: WorkerInfo -> IORef Count

321+ [A] [workerLatencyStart] :: WorkerInfo -> IORef (Count, AbsTime)

322+ [A] WorkerInfo :: Count -> IORef Count -> IORef (Count, AbsTime) -> WorkerInfo

323+ [A] Work

324+ [A] PartialWorker :: Count -> Work

325+ [A] ManyWorkers :: Int -> Count -> Work

326+ [A] BlockWait :: NanoSecond64 -> Work

327+ [A] ThreadAbort

328+ [A] ThreadAbort :: ThreadAbort

329+ [A] StopWhen

330+ [A] FirstStops :: StopWhen

331+ [A] AnyStops :: StopWhen

332+ [A] AllStop :: StopWhen

333+ [A] SVarStats

334+ [A] [totalDispatches] :: SVarStats -> IORef Int

335+ [A] [svarStopTime] :: SVarStats -> IORef (Maybe AbsTime)

336+ [A] [minWorkerLatency] :: SVarStats -> IORef NanoSecond64

337+ [A] [maxWorkers] :: SVarStats -> IORef Int

338+ [A] [maxWorkerLatency] :: SVarStats -> IORef NanoSecond64

339+ [A] [maxWorkQSize] :: SVarStats -> IORef Int

340+ [A] [maxOutQSize] :: SVarStats -> IORef Int

341+ [A] [maxHeapSize] :: SVarStats -> IORef Int

342+ [A] [avgWorkerLatency] :: SVarStats -> IORef (Count, NanoSecond64)

343+ [A] SVarStats :: IORef Int -> IORef Int -> IORef Int -> IORef Int -> IORef Int -> IORef (Count, NanoSecond64) -> IORef NanoSecond64 -> IORef NanoSecond64 -> IORef (Maybe AbsTime) -> SVarStats

344+ [A] Rate

345+ [A] [rateLow] :: Rate -> Double

346+ [A] [rateHigh] :: Rate -> Double

347+ [A] [rateGoal] :: Rate -> Double

348+ [A] [rateBuffer] :: Rate -> Int

349+ [A] Rate :: Double -> Double -> Double -> Int -> Rate

350+ [A] Limit

351+ [A] Unlimited :: Limit

352+ [A] Limited :: Word -> Limit

353+ [A] LatencyRange

354+ [A] [minLatency] :: LatencyRange -> NanoSecond64

355+ [A] [maxLatency] :: LatencyRange -> NanoSecond64

356+ [A] LatencyRange :: NanoSecond64 -> NanoSecond64 -> LatencyRange

357+ [A] Config

358+ [A] ChildEvent

359+ [A] ChildYield :: a -> ChildEvent a

360+ [A] ChildStopChannel :: ChildEvent a

361+ [A] ChildStop :: ThreadId -> Maybe SomeException -> ChildEvent a

362+ [A] Count

363+ [A] Count :: Int64 -> Count

364+ [A] workerRateControl :: Limit -> IORef Int -> YieldRateInfo -> WorkerInfo -> IO Bool

365+ [A] withDiagMVar :: Bool -> IO String -> String -> IO () -> IO ()

366+ [A] stopWhen :: StopWhen -> Config -> Config

367+ [A] setStreamLatency :: Int -> Config -> Config

368+ [A] sendYield :: Limit -> Limit -> IORef Int -> Maybe WorkerInfo -> Maybe YieldRateInfo -> IORef ([ChildEvent a], Int) -> MVar () -> ChildEvent a -> IO Bool

369+ [A] sendWithDoorBell :: IORef ([ChildEvent a], Int) -> MVar () -> ChildEvent a -> IO Int

370+ [A] sendStop :: IORef Int -> Maybe WorkerInfo -> Maybe YieldRateInfo -> IORef ([ChildEvent a], Int) -> MVar () -> IO ()

371+ [A] ringDoorBell :: IORef Bool -> MVar () -> IO ()

372+ [A] recordMaxWorkers :: MonadIO m => IORef Int -> SVarStats -> m ()

373+ [A] readOutputQRaw :: IORef ([ChildEvent a], Int) -> Maybe SVarStats -> IO ([ChildEvent a], Int)

374+ [A] readOutputQBasic :: IORef ([ChildEvent a], Int) -> IO ([ChildEvent a], Int)

375+ [A] rate :: Maybe Rate -> Config -> Config

376+ [A] printSVar :: IO String -> String -> IO ()

377+ [A] ordered :: Bool -> Config -> Config

378+ [A] newSVarStats :: IO SVarStats

379+ [A] newRateInfo :: Config -> IO (Maybe YieldRateInfo)

380+ [A] modifyThread :: MonadIO m => IORef (Set ThreadId) -> MVar () -> ThreadId -> m ()

381+ [A] minThreadDelay :: NanoSecond64

382+ [A] minRate :: Double -> Config -> Config

383+ [A] maxYields :: Maybe Int64 -> Config -> Config

384+ [A] maxThreads :: Int -> Config -> Config

385+ [A] maxRate :: Double -> Config -> Config

386+ [A] maxBuffer :: Int -> Config -> Config

387+ [A] magicMaxBuffer :: Word

388+ [A] isBeyondMaxRate :: Limit -> IORef Int -> YieldRateInfo -> IO Bool

389+ [A] interleaved :: Bool -> Config -> Config

390+ [A] inspect :: Bool -> Config -> Config

391+ [A] incrementYieldLimit :: Maybe (IORef Count) -> IO ()

392+ [A] handleChildException :: IORef ([ChildEvent a], Int) -> MVar () -> SomeException -> IO ()

393+ [A] getYieldLimit :: Config -> Maybe Count

394+ [A] getStreamRate :: Config -> Maybe Rate

395+ [A] getStreamLatency :: Config -> Maybe NanoSecond64

396+ [A] getStopWhen :: Config -> StopWhen

397+ [A] getOrdered :: Config -> Bool

398+ [A] getMaxThreads :: Config -> Limit

399+ [A] getMaxBuffer :: Config -> Limit

400+ [A] getInterleaved :: Config -> Bool

401+ [A] getInspectMode :: Config -> Bool

402+ [A] getEagerDispatch :: Config -> Bool

403+ [A] getBound :: Config -> Bool

404+ [A] estimateWorkers :: Limit -> Count -> Count -> NanoSecond64 -> NanoSecond64 -> NanoSecond64 -> LatencyRange -> Work

405+ [A] eager :: Bool -> Config -> Config

406+ [A] dumpWorkerCount :: Show a => IORef a -> IO String

407+ [A] dumpSVarStats :: Bool -> Maybe YieldRateInfo -> SVarStats -> IO String

408+ [A] dumpRunningThreads :: Show a => IORef a -> IO String

409+ [A] dumpOutputQ :: (Foldable t, Show a1) => IORef (t a2, a1) -> IO String

410+ [A] dumpNeedDoorBell :: Show a => IORef a -> IO String

411+ [A] dumpDoorBell :: Show a => MVar a -> IO String

412+ [A] dumpCreator :: Show a => a -> String

413+ [A] delThread :: MonadIO m => IORef (Set ThreadId) -> ThreadId -> m ()

414+ [A] defaultConfig :: Config

415+ [A] decrementYieldLimit :: Maybe (IORef Count) -> IO Bool

416+ [A] constRate :: Double -> Config -> Config

417+ [A] collectLatency :: Bool -> SVarStats -> YieldRateInfo -> Bool -> IO (Count, AbsTime, NanoSecond64)

418+ [A] cleanupSVar :: IORef (Set ThreadId) -> IO ()

419+ [A] boundThreads :: Bool -> Config -> Config

420+ [A] avgRate :: Double -> Config -> Config

421+ [A] allThreadsDone :: MonadIO m => IORef (Set ThreadId) -> m Bool

422+ [A] addThread :: MonadIO m => IORef (Set ThreadId) -> ThreadId -> m ()

423+[C] Streamly.Internal.Control.ForkLifted

424+ [A] doForkWith :: MonadRunInIO m => Bool -> m () -> RunInIO m -> (SomeException -> IO ()) -> m ThreadId

docs/User/Project/CONTRIBUTORS.md+8

@@ -4,6 +4,14 @@ please see https://github.com/composewell/streamly/issues.

44Use `git shortlog -sn tag1...tag2` on the git repository to get a list of

55contributors between two repository tags.

7+## 0.10.0

9+* Adithya Kumar

10+* Harendra Kumar

11+* Ranjeet Kumar Ranjan

12+* Shlok Datye

13+* Bodigrim

14+

715## 0.9.0

816

917* Harendra Kumar

docs/User/Project/Changelog.md+25−10

@@ -2,26 +2,40 @@

33

5−## Unreleased

5+## 0.10.1 (Jan 2024)

7+* Fix TH macros in `Streamly.Data.Stream.MkType` for GHC 9.6 and above.

9+## 0.10.0 (Nov 2023)

10+

11+See

12+[0.9.0-0.10.0 API Changelog](https://github.com/composewell/streamly/blob/streamly-0.10.0/docs/User/Project/ApiChangelogs/0.9.0-0.10.0.txt)

13+for a full list of API changes in this release. Only a few significant

14+changes are mentioned here. For changes to the core functionality

15+please see the changelog of the `streamly-core` package.

616

717### Breaking Changes

818

9−* MonadTrans and MonadBase instances has been removed for `AsyncT`,

19+* `MonadTrans` and `MonadBase` instances have been removed for `AsyncT`,

1020 `ParallelT`, `AheadT` for GHC versions 9.6 onwards. This is due to a

1121 breaking change in `transformers` 0.6. You can replace `lift` with

1222 `fromEffect` when using these as top level monads in a monad stack.

1323

14−* Rethrow the exception promptly in bracketIO.

24+### Enhancements

25+

26+* __Concurrent Folds__: `Streamly.Data.Fold.Prelude` module added with

27+ concurrent fold APIs and a container fold API to fold a key value

28+ stream to a HashMap.

1529

1630## 0.9.0 (Mar 2023)

1731

1832Also see the following:

1933

20−* [streamly-0.9.0 Upgrade Guide](/docs/User/ProjectRelated/Upgrading.md) or

34+* [streamly-0.9.0 Upgrade Guide](https://github.com/composewell/streamly/blob/streamly-0.10.0/docs/User/Project/Upgrading-0.8-to-0.9.md) or

2135 https://hackage.haskell.org/package/streamly-0.9.0/docs/docs/User/ProjectRelated/Upgrading.md

22−* [streamly-0.9.0 API Changelog](/docs/User/ProjectRelated/ApiChangelogs/0.8.3-0.9.0.txt) or

36+* [streamly-0.9.0 API Changelog](https://github.com/composewell/streamly/blob/streamly-0.10.0/docs/User/Project/ApiChangelogs/0.8.3-0.9.0.txt) or

2337 https://hackage.haskell.org/package/streamly-0.9.0/docs/docs/User/ProjectRelated/ApiChangelogs/0.8.3-0.9.0.txt

24−* [streamly-core-0.1.0 API Changelog](/core/docs/ApiChangelogs/0.1.0.txt) or

38+* [streamly-core-0.1.0 API Changelog](https://github.com/composewell/streamly/blob/streamly-0.10.0/core/docs/ApiChangelogs/0.1.0.txt) or

2539 https://hackage.haskell.org/package/streamly-core-0.1.0/docs/docs/ApiChangelogs/0.1.0.txt

2640

2741### Package split

@@ -68,8 +82,9 @@ added which provide a CPS based stream implementation. `Stream` and

6882`StreamK` types can be easily interconverted.

6983

7084The old code can be adapted to use the new modules with some changes.

71−See the [upgrade guide](/docs/User/ProjectRelated/Upgrading.md) for more

72−details on how to adapt your existing code to the new release.

85+See the

86+[streamly-0.9.0 Upgrade Guide](https://github.com/composewell/streamly/blob/streamly-0.10.0/docs/User/Project/Upgrading-0.8-to-0.9.md)

87+for more details on how to adapt your existing code to the new release.

7388

7489### Enhancements

7590

@@ -150,7 +165,7 @@ See ApiChangelogs/0.8.3.txt for new APIs introduced.

150165

151166## 0.8.0 (Jun 2021)

152167

153−See [API Changelog](/docs/User/ProjectRelated/ApiChangelogs/0.8.3.txt) for a complete list of signature

168+See [API Changelog](/docs/User/Project/ApiChangelogs/0.8.3.txt) for a complete list of signature

154169changes and new APIs introduced.

155170

156171### Breaking changes

@@ -189,7 +204,7 @@ changes and new APIs introduced.

189204

190205### Enhancements

191206

192−* See [API Changelog](/docs/User/ProjectRelated/ApiChangelogs/0.8.3.txt) for a complete list of new

207+* See [API Changelog](/docs/User/Project/ApiChangelogs/0.8.3.txt) for a complete list of new

193208 modules and APIs introduced.

194209* The Fold type is now more powerful, the new termination behavior allows

195210 to express basic parsing of streams using folds.

docs/User/Project/README.md+9−2

@@ -1,8 +1,15 @@

11# [Streamly][]: Idiomatic Haskell with C-Like Performance

33[![Gitter chat](https://badges.gitter.im/composewell/gitter.svg)](https://gitter.im/composewell/streamly)

4+[![Hackage](https://img.shields.io/hackage/v/streamly-core.svg?style=flat)](https://hackage.haskell.org/package/streamly-core)

45[![Hackage](https://img.shields.io/hackage/v/streamly.svg?style=flat)](https://hackage.haskell.org/package/streamly)

7+## Upgrading to 0.9.0+

9+Please read the [Streamly 0.9.0 Upgrade Guide](/docs/User/Project/Upgrading-0.8-to-0.9.md).

10+

11+## Overview

12+

613Streamly is a powerful Haskell library that provides developers with

714the essential building blocks to create safe, scalable, modular, and

815high-performance software. With Streamly, developers can enjoy the

@@ -17,8 +24,8 @@ comprising:

1724* Fearless, declarative concurrency for seamless parallel execution.

1825* A collection of ecosystem libraries for fast and efficient development.

1926

20−Check out the [Streamly Setup and Usage

21−Guide](/docs/User/Tutorials/setup-and-usage.md) and [Quick

27+Check out the [Streamly Getting Started

28+Guide](/docs/User/Tutorials/getting-started.md) and [Quick

2229Overview](/docs/User/Tutorials/quick-overview.md) for an introduction

2330to the library. For more detailed documentation, visit the [Haskell

2431Streamly website][Streamly].

docs/User/Project/Upgrading.md → docs/User/Project/Upgrading-0.8-to-0.9.mdrenamed+87−9

@@ -1,9 +1,9 @@

11# Upgrading to streamly 0.9.0

3−Also see:

3+Also see the detailed changelog describing all the changes in the release:

5−* [0.9.0 API Changelog](/docs/User/ProjectRelated/ApiChangelogs/0.8.3-0.9.0.txt) or

6− https://hackage.haskell.org/package/streamly-0.9.0/docs/docs/User/ProjectRelated/ApiChangelogs/0.8.3-0.9.0.txt

5+* [0.9.0 API Changelog](/docs/User/Project/ApiChangelogs/0.8.3-0.9.0.txt) or

6+* [0.9.0 API Changelog](https://hackage.haskell.org/package/streamly-0.9.0/docs/docs/User/ProjectRelated/ApiChangelogs/0.8.3-0.9.0.txt)

88`Streamly.Prelude` module has been deprecated, equivalent

99functionality is covered by the `Streamly.Data.Stream`,

@@ -46,13 +46,14 @@ Assume the following imports in the code snippets below:

4646

4747## Dependencies in cabal file

4848

49−If you allow streamly < 0.9 in version bounds then you may not be able

50−to build if you:

49+If you allow streamly < 0.9 in version bounds in your package then you

50+may not be able to build if any of the following is true:

5151

52−* also depend on the new `streamly-core` package (module name conflicts)

53−* depend on `streamly-bytestring` >= 0.2 (type conflict)

52+* there is a dependency on `streamly-core` (module name conflicts)

53+* there is a dependency on `streamly-bytestring` >= 0.2 (type conflict)

5454

55−You can use a build flag to allow older streamly versions:

55+You can define and use a build flag in your package to allow older

56+streamly versions:

5657

5758```

5859flag old-streamly

@@ -103,6 +104,82 @@ ZipSerialM or WSerialT) with custom Applicative or Monadic properties.

103104But in general, try to avoid specific types and use explicit functions

104105from the stream module.

105106

107+Here are the TH macro based recipes to define types equivalent to all the older

108+types:

109+

110+```

111+{-# LANGUAGE StandaloneDeriving #-}

112+{-# LANGUAGE TemplateHaskell #-}

113+{-# LANGUAGE TypeFamilies #-}

114+{-# LANGUAGE UndecidableInstances #-}

115+

116+import Streamly.Data.Stream.MkType

117+import qualified Streamly.Data.Stream.Prelude as Stream

118+import qualified Streamly.Internal.Data.StreamK as StreamK

119+import Language.Haskell.TH

120+```

121+

122+`SerialT`: Use `CrossStreamK` from `Streamly.Internal.Data.StreamK` equivalent

123+Monad and other instances.

124+

125+For `WSerialT`, create a newtype wrapper using the following monad bind

126+operation:

127+

128+```

129+bind = StreamK.bindWith StreamK.interleave

130+```

131+

132+`WSerialT` requires `StreamK`, therefore you can not generate the code directly

133+using TH macros. However, you can print the code generated by the TH macro.

134+Then you can tweak that code, replacing `Stream` type with `StreamK`.

135+

136+```

137+expr <- runQ (mkCrossType "WSerialT" "bind" True)

138+putStrLn $ pprint expr

139+```

140+

141+For `AsyncT`:

142+

143+```

144+bind = flip (Stream.parConcatMap id)

145+$(mkCrossType "AsyncT" "bind" True)

146+```

147+

148+For `WAsyncT`:

149+

150+```

151+bind = flip (Stream.parConcatMap (Stream.interleaved True))

152+$(mkCrossType "WAsyncT" "bind" True)

153+```

154+

155+For `AheadT`:

156+

157+```

158+bind = flip (Stream.parConcatMap (Stream.ordered True))

159+$(mkCrossType "AheadT" "bind" True)

160+```

161+

162+For `ParallelT`:

163+

164+```

165+bind = flip (Stream.parConcatMap (Stream.eager True))

166+$(mkCrossType "ParallelT" "bind" True)

167+```

168+

169+For `ZipSerialM`:

170+

171+```

172+apply = Stream.zipWith ($)

173+$(mkZipType "ZipSerialM" "apply" False)

174+```

175+

176+For `ZipAsync`:

177+

178+```

179+apply = Stream.parApply id

180+$(mkZipType "ZipAsync" "apply" False)

181+```

182+

106183`adapt` is not needed anymore.

107184

108185```haskell docspec

@@ -228,7 +305,8 @@ combinators are prefixed with `par`.

228305Parallel combinators take a concurrency config argument to specify the

229306concurrency control parameters. The following combinators have the same

230307meaning as before except that they are used to set the config parameters

231−instead of being applied on the stream.

308+instead of being applied on the stream:

309+

232310* `rate`

233311* `maxRate`

234312* `constRate`

docs/User/Tutorials/quick-overview.md+10−14

@@ -44,8 +44,6 @@ The meaning of these types are:

4444 `a` in the IO Monad.

4545* A `Fold IO a b` is a representation of a function that converts a stream of

4646 type `a` to a final accumulator of type `b` in the IO Monad.

47−* An `Unfold IO a b` is a representation of a function that converts a seed

48− value of type `a` into a stream of values of type `b` in the IO Monad.

4947

5048## The Examples

5149

@@ -219,7 +217,7 @@ import qualified Streamly.Unicode.Stream as Unicode

219217

220218countArray :: Array Word8 -> IO Counts

221219countArray arr =

222− Stream.unfold Array.reader arr -- Stream IO Word8

220+ Array.read arr -- Stream IO Word8

223221 & Unicode.decodeLatin1 -- Stream IO Char

224222 & Stream.fold (Fold.foldl' count (Counts 0 0 0 True)) -- IO Counts

225223```

@@ -335,9 +333,9 @@ fetch w = threadDelay 1000000 >> return (w,w)

335333-- connection is closed.

336334lookupWords :: Socket -> IO ()

337335lookupWords sk =

338− Stream.unfold Socket.reader sk -- Stream IO Word8

336+ Socket.read sk -- Stream IO Word8

339337 & Unicode.decodeLatin1 -- Stream IO Char

340− & Stream.parseMany word -- Stream IO String

338+ & Stream.wordsBy isSpace Fold.toList -- Stream IO String

341339 & Stream.parMapM cfg fetch -- Stream IO (String, String)

342340 & fmap show -- Stream IO String

343341 & Stream.intersperse "\n" -- Stream IO String

@@ -346,7 +344,6 @@ lookupWords sk =

346344

347345 where

348346

349− word = Parser.wordBy isSpace Fold.toList

350347 cfg = Stream.ordered True

351348

352349serve :: Socket -> IO ()

@@ -357,9 +354,9 @@ serve sk = finally (lookupWords sk) (close sk)

357354-- "nc" as a client to try it out.

358355main :: IO ()

359356main =

360− Stream.unfold TCP.acceptorOnPort 8091 -- Stream IO Socket

361− & Stream.parMapM id serve -- Stream IO ()

362− & Stream.fold Fold.drain -- IO ()

357+ TCP.accept 8091 -- Stream IO Socket

358+ & Stream.parMapM id serve -- Stream IO ()

359+ & Stream.fold Fold.drain -- IO ()

363360```

364361

365362### Merging Incoming Streams

@@ -379,7 +376,6 @@ including the imports that we have omitted below.

379376import Streamly.Data.Stream (Stream)

380377import System.IO (IOMode(AppendMode), Handle, withFile)

381378

382−import qualified Streamly.Data.Unfold as Unfold

383379import qualified Streamly.Network.Socket as Socket

384380import qualified Streamly.FileSystem.Handle as Handle

385381

@@ -388,9 +384,9 @@ import qualified Streamly.FileSystem.Handle as Handle

388384-- buffering for safety.

389385readLines :: Socket -> Stream IO (Array Char)

390386readLines sk =

391− Stream.unfold Socket.reader sk -- Stream IO Word8

392− & Unicode.decodeLatin1 -- Stream IO Char

393− & Stream.foldMany line -- Stream IO (Array Char)

387+ Socket.read sk -- Stream IO Word8

388+ & Unicode.decodeLatin1 -- Stream IO Char

389+ & Stream.foldMany line -- Stream IO (Array Char)

394390

395391 where

396392

@@ -405,7 +401,7 @@ recv sk = Stream.finallyIO (close sk) (readLines sk)

405401-- streams at line boundaries and writes the merged stream to a file.

406402server :: Handle -> IO ()

407403server file =

408− Stream.unfold TCP.acceptorOnPort 8090 -- Stream IO Socket

404+ TCP.accept 8090 -- Stream IO Socket

409405 & Stream.parConcatMap (Stream.eager True) recv -- Stream IO (Array Char)

410406 & Stream.unfoldMany Array.reader -- Stream IO Char

411407 & Unicode.encodeLatin1 -- Stream IO Word8

src/Streamly.hsdeleted−625

@@ -1,625 +0,0 @@

1−{-# OPTIONS_GHC -Wno-deprecations #-}

2−

3−-- |

4−-- Module : Streamly

6−--

7−-- License : BSD3

8−-- Maintainer : streamly@composewell.com

9−-- Stability : experimental

10−-- Portability : GHC

11−--

12−-- Streamly is a general purpose programming framework using cocnurrent data

13−-- flow programming paradigm. It can be considered as a generalization of

14−-- Haskell lists to monadic streaming with concurrent composition capability.

15−-- The serial stream type in streamly @SerialT m a@ is like the list type @[a]@

16−-- parameterized by the monad @m@. For example, @SerialT IO a@ is a moral

17−-- equivalent of @[a]@ in the IO monad. Streams are constructed very much like

18−-- lists, except that they use 'nil' and 'cons' instead of '[]' and ':'.

19−--

20−-- @

21−-- > import "Streamly"

22−-- > import "Streamly.Prelude" (cons, consM)

23−-- > import qualified "Streamly.Prelude" as S

24−-- >

25−-- > S.toList $ 1 \`cons` 2 \`cons` 3 \`cons` nil

26−-- [1,2,3]

27−-- @

28−--

29−-- Unlike lists, streams can be constructed from monadic effects:

30−--

31−-- @

32−-- > S.'toList' $ 'getLine' \`consM` 'getLine' \`consM` S.'nil'

33−-- hello

34−-- world

35−-- ["hello","world"]

36−-- @

37−--

38−-- Streams are processed just like lists, with list like combinators, except

39−-- that they are monadic and work in a streaming fashion. Here is a simple

40−-- console echo program example:

41−--

42−-- @

43−-- > S.drain $ S.repeatM getLine & S.mapM putStrLn

44−-- @

45−--

46−-- @SerialT Identity a@ is a moral equivalent of pure lists. Streamly utilizes

47−-- fusion for high performance, therefore, we can represent and process strings

48−-- as streams of 'Char', encode and decode the streams to/from UTF8 and

49−-- serialize them to @Array Word8@ obviating the need for special purpose

50−-- libraries like @bytestring@ and @text@.

51−--

52−-- For more details

53−-- please see the "Streamly.Tutorial" module and the examples directory in this

54−-- package.

55−

56−{-# LANGUAGE NoMonomorphismRestriction #-}

57−

58−#include "inline.hs"

59−

60−module Streamly

61−{-# DEPRECATED "Please use \"Streamly.Data.Stream.Prelude\" from streamly package and \"Streamly.Data.Fold\" from streamly-core package instead." #-}

62− (

63− -- -- * Concepts Overview

64− -- -- ** Streams

65− -- -- $streams

66−

67− -- -- ** Folds

68− -- -- $folds

69−

70− -- -- ** Arrays

71− -- -- $arrays

72−

73− -- * Module Overview

74− -- $streamtypes

75−

76− -- * Type Synonyms

77− MonadAsync

78−

79− -- * Stream transformers

80− -- | A stream represents a sequence of pure or effectful actions. The

81− -- `cons` and `consM` operations and the corresponding operators '.:' and

82− -- '|:' can be used to join pure values or effectful actions in a sequence.

83− -- The effects in the stream can be executed in many different ways

84− -- depending on the type of stream. In other words, the behavior of 'consM'

85− -- depends on the type of the stream.

86− --

87− -- There are three high level categories of streams, /spatially ordered/

88− -- streams, /speculative/ streams and /time ordered/ streams. Spatially

89− -- ordered streams, 'SerialT' and 'WSerialT', execute the effects in serial

90− -- order i.e. one at a time and present the outputs of those effects to the

91− -- consumer in the same order. Speculative streams, 'AheadT', may execute

92− -- many effects concurrently but present the outputs to the consumer in the

93− -- specified spatial order. Time ordered streams, 'AsyncT', 'WAsyncT' and

94− -- 'ParallelT', may execute many effects concurrently and present the

95− -- outputs of those effects to the consumer in time order i.e. as soon as

96− -- the output is generated.

97− --

98− -- We described above how the effects in a sequence are executed for

99− -- different types of streams. The behvavior of the 'Semigroup' and 'Monad'

100− -- instances follow the behavior of 'consM'. Stream generation operations

101− -- like 'repeatM' also execute the effects differently for different

102− -- streams, providing a concurrent generation capability when used with

103− -- stream types that execute effects concurrently. Similarly, effectful

104− -- transformation operations like 'mapM' also execute the transforming

105− -- effects differently for different types of streams.

106−

107− -- ** Serial Streams

108− -- $serial

109− , SerialT

110− , WSerialT

111−

112− -- ** Speculative Streams

113− -- $ahead

114− , AheadT

115−

116− -- ** Asynchronous Streams

117− -- $async

118− , AsyncT

119− , WAsyncT

120− , ParallelT

121−

122− -- ** Zipping Streams

123− -- $zipping

124− , ZipSerialM

125− , ZipAsyncM

126−

127− -- * Parallel Function Application

128− -- $application

129− , (IP.|$)

130− , (IP.|&)

131− , (IP.|$.)

132− , (IP.|&.)

133− , mkAsync

134−

135− -- * Merging Streams

136− -- $sum

137− , serial

138− , wSerial

139− , ahead

140− , async

141− , wAsync

142− , parallel

143−

144− -- * Concurrency Control

145− -- $concurrency

146− , maxThreads

147− , maxBuffer

148−

149− -- * Rate Limiting

150− , Rate (..)

151− , rate

152− , avgRate

153− , minRate

154− , maxRate

155− , constRate

156−

157− -- * Stream Type Adapters

158− -- $adapters

159− , IsStream ()

160−

161− , serially

162− , wSerially

163− , asyncly

164− , aheadly

165− , wAsyncly

166− , parallely

167− , zipSerially

168− , zipAsyncly

169− , adapt

170−

171− -- * IO Streams

172− , Serial

173− , WSerial

174− , Ahead

175− , Async

176− , WAsync

177− , Parallel

178− , ZipSerial

179− , ZipAsync

180−

181− -- ** Folding Containers of Streams

182− -- | These are variants of standard 'Foldable' fold functions that use a

183− -- polymorphic stream sum operation (e.g. 'async' or 'wSerial') to fold a

184− -- finite container of streams. Note that these are just special cases of

185− -- the more general 'concatMapWith' operation.

186− --

187− , foldWith

188− , foldMapWith

189− , forEachWith

190−

191− -- * Re-exports

192− , Semigroup (..)

193− -- * Deprecated

194− , runStream

195− )

196−where

197−

198−import Data.Semigroup (Semigroup(..))

199−import Streamly.Internal.Control.Concurrent (MonadAsync)

200−import Streamly.Internal.Data.SVar (Rate(..))

201−import Streamly.Internal.Data.Stream.Ahead

202−import Streamly.Internal.Data.Stream.Async

203−import Streamly.Internal.Data.Stream.IsStream.Combinators

204−import Streamly.Internal.Data.Stream.IsStream.Expand

205−import Streamly.Internal.Data.Stream.IsStream.Type

206−

207−import qualified Streamly.Prelude as P

208−import qualified Streamly.Internal.Data.Stream.IsStream as IP

209−import qualified Streamly.Internal.Data.Stream.IsStream.Transform as Transform

210−

211−-- XXX provide good succinct examples of pipelining, merging, splitting etc.

212−-- below.

213−--

214−-- $streams

215−--

216−-- A program is expressed as a network of streams and folds. A stream is a

217−-- source or generator of data elements and a fold is a consumer of data elements that

218−-- reduces multiple input elements to a single value.

219−--

220−-- In the following example, a 'Word8' stream is generated by using

221−-- 'Streamly.FileSystem.Handle.read' on a file handle, then the

222−-- 'Streamly.Prelude.splitBySuffix' transformation splits the stream on

223−-- newlines (ascii value 10); it uses the 'Streamly.Data.Fold.drain' fold to reduce

224−-- the resulting lines to unit values (@()@), 'Streamly.Prelude.length' fold

225−-- then counts the unit elements in the resulting stream which gives us the

226−-- number of lines in the file:

227−--

228−-- > S.length $ S.splitOnSuffix FL.drain 10 $ FH.read fh

229−--

230−-- The following example folds the lines to arrays of 'Word8' using the

231−-- 'Streamly.Data.Array.Foreign.writeF' fold and then wraps the lines in square

232−-- brackets before writing them to standard output using

233−-- 'Streamly.FileSystem.Handle.write':

234−--

235−-- > wrapLine ln = S.fromList "[" <> A.read ln <> S.fromList "]\n"

236−-- > readLines = S.splitOnSuffix A.writeF 10

237−-- > FH.write stdout $ S.concatMap wrapLine $ readLines fh1

238−--

239−-- One stream can be appended after another:

240−--

241−-- > FH.write stdout $ S.concatMap wrapLine $ readLines fh1 <> readLines fh2

242−--

243−-- The following example reads two files concurrently, merges the lines from

244−-- the two streams and writes the resulting stream to another file:

245−--

246−-- > FH.write stdout $ S.concatMap wrapLine $ readLines fh1 `parallel` readLines fh2

247−--

248−-- There are many ways to generate, merge, zip, transform and fold data

249−-- streams. Many transformations can be chained in a stream pipeline. See

250−-- "Streamly.Prelude" module for combinators to manipulate streams.

251−

252−-- $folds

253−--

254−-- The way stream types in this module like 'SerialT' represent data sources,

255−-- the same way the 'Fold' type from "Streamly.Data.Fold" represents data sinks or

256−-- reducers of streams. Reducers can be combined to consume a stream source in

257−-- many ways. The simplest is to reduce a stream source using a fold e.g.:

258−--

259−-- > S.fold FL.length $ S.enumerateTo 100

260−--

261−-- Folds are consumers of streams and can be used to split a stream into

262−-- multiple independent flows. Grouping transforms a stream by applying a fold

263−-- on segments of a stream, distributing applies multiple folds in parallel on

264−-- the same stream and combines them, partitioning sends different elements of

265−-- a stream to different folds, unzipping divides the elements of a stream into

266−-- parts and sends them through different folds. Parsers are nothing but a

267−-- particular type of folds. Transformations can be applied contravariantly on

268−-- the input of a fold.

269−--

270−-- @

271−--

272−-- |---transform----Fold m a b--------|

273−-- ---stream m a-->transform-->| |---f b c ...

274−-- |---transform----Fold m a c--------|

275−-- | |

276−-- ...

277−-- @

278−--

279−

280−-- $arrays

281−--

282−-- Streamly arrays (See "Streamly.Data.Array.Foreign") complement streams to provide an

283−-- efficient computing paradigm. Streams are suitable for immutable

284−-- transformations of /potentially infinite/ data using /sequential access/ and

285−-- pipelined transformations whereas arrays are suitable for in-place

286−-- transformations of /necessarily finite/ data using /random access/. Streams

287−-- are synonymous with /sequential pipelined processing/ whereas arrays are

288−-- synonymous with /efficient buffering and random access/.

289−--

290−-- In general, a data processing pipeline reads data from some IO device, does

291−-- some processing on it and finally writes the output to another IO device.

292−-- Streams provide the overall framework of sequential processing pipeline in

293−-- which arrays are used as buffering elements in the middle. In addition to

294−-- buffering in the middle, arrays can also be used at the boundaries of the

295−-- pipeline to efficiently interface with external storage systems like memory,

296−-- files and network. If streams are the pipes in a water pipeline network

297−-- then arrays are like the storage tanks in the middle. On the input side,

298−-- think of arrays as buckets to fetch water to feed the pipeline and on the

299−-- output side buckets to remove the processed water.

300−--

301−-- 'ByteString' data type from the 'bytestring' package and the 'Text' data

302−-- type from the 'text' package are special cases of arrays. 'ByteString' is

303−-- like @Array Word8@ and 'Text' is like @utf16@ encoded @Array Word8@.

304−-- Streamly arrays can be transformed as efficiently as @bytestring@ or @text@

305−-- by using stream operations on them.

306−

307−-- Streams and arrays are equally important in computing. They are computing

308−-- duals of each other.

309−

310−-- $streamtypes

311−-- The basic stream type is 'Serial', it represents a sequence of IO actions,

312−-- and is a 'Monad'. The type 'SerialT' is a monad transformer that can

313−-- represent a sequence of actions in an arbitrary monad. The type 'Serial' is

314−-- in fact a synonym for @SerialT IO@. There are a few more types similar to

315−-- 'SerialT', all of them represent a stream and differ only in the

316−-- 'Semigroup', 'Applicative' and 'Monad' compositions of the stream. 'Serial'

317−-- and 'WSerial' types compose serially whereas 'Async' and 'WAsync'

318−-- types compose concurrently. All these types can be freely inter-converted

319−-- using type combinators without any cost. You can freely switch to any type

320−-- of composition at any point in the program. When no type annotation or

321−-- explicit stream type combinators are used, the default stream type is

322−-- inferred as 'Serial'.

323−--

324−-- This module exports stream types, instances and combinators for:

325−--

326−-- * converting between different stream types

327−-- * appending and concurrently merging streams

328−-- * Concurrency control

329−-- * Concurrent function application

330−-- * Stream rate control

331−--

332−-- This module is designed to be imported unqualified:

333−--

334−-- @

335−-- import Streamly

336−-- @

337−--

338−-- See the "Streamly.Prelude" module for APIs for construction,

339−-- generation, elimination and transformation of streams.

340−

341−------------------------------------------------------------------------------

342−-- Eliminating a stream

343−------------------------------------------------------------------------------

344−

345−-- | Same as "Streamly.Prelude.runStream".

346−--

347−{-# DEPRECATED runStream "Please use Streamly.Prelude.drain instead." #-}

348−runStream :: Monad m => SerialT m a -> m ()

349−runStream = P.drain

350−

351−{-

352−-- | Same as "Streamly.Prelude.foldWith".

353−--

354−{-# DEPRECATED foldWith "Please use Streamly.Prelude.foldWith instead." #-}

355−{-# INLINABLE foldWith #-}

356−foldWith :: (IsStream t, Foldable f)

357− => (t m a -> t m a -> t m a) -> f (t m a) -> t m a

358−foldWith = P.foldWith

359−

360−-- | Same as "Streamly.Prelude.foldMapWith".

361−--

362−{-# DEPRECATED foldMapWith "Please use Streamly.Prelude.foldMapWith instead." #-}

363−{-# INLINABLE foldMapWith #-}

364−foldMapWith :: (IsStream t, Foldable f)

365− => (t m b -> t m b -> t m b) -> (a -> t m b) -> f a -> t m b

366−foldMapWith = P.foldMapWith

367−

368−-- | Same as "Streamly.Prelude.forEachWith".

369−--

370−{-# DEPRECATED forEachWith "Please use Streamly.Prelude.forEachWith instead." #-}

371−{-# INLINABLE forEachWith #-}

372−forEachWith :: (IsStream t, Foldable f)

373− => (t m b -> t m b -> t m b) -> f a -> (a -> t m b) -> t m b

374−forEachWith = P.forEachWith

375−-}

376−

377−-- XXX Deprecate it in 0.8.0

378−--

379−-- | Make a stream asynchronous, triggers the computation and returns a stream

380−-- in the underlying monad representing the output generated by the original

381−-- computation. The returned action is exhaustible and must be drained once. If

382−-- not drained fully we may have a thread blocked forever and once exhausted it

383−-- will always return 'empty'.

384−--

385−-- @since 0.2.0

386−{-# INLINABLE mkAsync #-}

387−mkAsync :: (IsStream t, MonadAsync m) => t m a -> m (t m a)

388−mkAsync = return . Transform.mkAsync

389−

390−------------------------------------------------------------------------------

391−-- Documentation

392−------------------------------------------------------------------------------

393−

394−-- $serial

395−--

396−-- When a stream consumer demands an element from a serial stream constructed

397−-- as @a \`consM` b \`consM` ... nil@, the action @a@ at the head of the stream

398−-- sequence is executed and the result is supplied to the consumer. When the

399−-- next element is demanded, the action @b@ is executed and its result is

400−-- supplied. Thus, the effects are performed and results are consumed strictly

401−-- in a serial order. Serial streams can be considered as /spatially ordered/

402−-- streams as the order of execution and consumption is the same as the spatial

403−-- order in which the actions are composed by the programmer.

404−--

405−-- Serial streams enforce the side effects as well as the results of the

406−-- actions to be in the same order in which the actions are added to the

407−-- stream. Therefore, the semigroup operation for serial streams is not

408−-- commutative:

409−--

410−-- @

411−-- a <> b is not the same as b <> a

412−-- @

413−--

414−-- There are two serial stream types 'SerialT' and 'WSerialT'. The stream

415−-- evaluation of both the variants works in the same way as described above,

416−-- they differ only in the 'Semigroup' and 'Monad' implementaitons.

417−

418−-- $ahead

419−--

420−-- When a stream consumer demands an element from a speculative stream

421−-- constructed as @a \`consM` b \`consM` ... nil@, the action @a@ at the head

422−-- of the stream is executed and the output of the action is supplied to the

423−-- consumer. However, in addition to the action at the head multiple actions

424−-- following it may also be executed concurrently and the results buffered.

425−-- When the next element is demanded it may be served from the buffer and we

426−-- may execute the next action in the sequence to keep the buffer adequately

427−-- filled. Thus, the actions are executed concurrently but results consumed in

428−-- serial order just like serial streams. `consM` can be used to fold an

429−-- infinite lazy container of effects, as the number of concurrent executions

430−-- is limited.

431−--

432−-- Similar to 'consM', the monadic stream generation (e.g. replicateM) and

433−-- transformation operations (e.g. mapM) on speculative streams can execute

434−-- multiple effects concurrently in a speculative manner.

435−--

436−-- How many effects can be executed concurrently and how many results can be

437−-- buffered are controlled by 'maxThreads' and 'maxBuffer' combinators

438−-- respectively. The actual number of concurrent threads is adjusted according

439−-- to the rate at which the consumer is consuming the stream. It may even

440−-- execute actions serially in a single thread if that is enough to match the

441−-- consumer's speed.

442−--

443−-- Speculative streams enforce ordering of the results of actions in the stream

444−-- but the side effects are only partially ordered. Therefore, the semigroup

445−-- operation for speculative streams is not commutative from the pure outputs

446−-- perspective but commutative from side effects perspective.

447−

448−-- $async

449−--

450−-- /Scheduling and execution:/ In an asynchronous stream @a \`consM` b \`consM`

451−-- c ...@, the actions @a@, @b@, and @c@ are executed concurrently with the

452−-- consumer of the stream. The actions are /scheduled/ for execution in the

453−-- same order as they are specified in the stream. Multiple scheduled actions

454−-- may be /executed/ concurrently in parallel threads of execution. The

455−-- actions may be executed out of order and they may complete at arbitrary

456−-- times. Therefore, the /effects/ of the actions may be observed out of

457−-- order.

458−--

459−-- /Buffering:/ The /results/ from multiple threads of execution are queued in

460−-- a buffer as soon as they become available. The consumer of the stream is

461−-- served from this buffer. Therefore, the consumer may observe the results to

462−-- be out of order. In other words, an asynchronous stream is an unordered

463−-- stream i.e. order does not matter.

464−--

465−-- /Concurrency control:/ Threads are suspended if the `maxBuffer` limit is

466−-- reached, and resumed when the consumer makes space in the buffer. The

467−-- maximum number of concurrent threads depends on `maxThreads`. Number of

468−-- threads is increased or decreased based on the speed of the consumer.

469−--

470−-- /Generation operations:/ Concurrent stream generation operations e.g.

471−-- 'Streamly.Prelude.replicateM' when used in async style schedule and execute

472−-- the stream generating actions in the manner described above. The generation

473−-- actions run concurrently, effects and results of the actions as observed by

474−-- the consumer of the stream may be out of order.

475−--

476−-- /Transformation operations:/ Concurrent stream transformation operations

477−-- e.g. 'Streamly.Prelude.mapM', when used in async style, schedule and

478−-- execute transformation actions in the manner described above. Transformation

479−-- actions run concurrently, effects and results of the actions may be

480−-- observed by the consumer out of order.

481−--

482−-- /Variants:/ There are two asynchronous stream types 'AsyncT' and 'WAsyncT'.

483−-- They are identical with respect to single stream evaluation behavior. Their

484−-- behaviors differ in how they combine multiple streams using 'Semigroup' or

485−-- 'Monad' composition. Since the order of elements does not matter in

486−-- asynchronous streams the 'Semigroup' operation is effectively commutative.

487−

488−-- $zipping

489−--

490−-- 'ZipSerialM' and 'ZipAsyncM', provide 'Applicative' instances for zipping the

491−-- corresponding elements of two streams together. Note that these types are

492−-- not monads.

493−

494−-- $application

495−--

496−-- Stream processing functions can be composed in a chain using function

497−-- application with or without the '$' operator, or with reverse function

498−-- application operator '&'. Streamly provides concurrent versions of these

499−-- operators applying stream processing functions such that each stage of the

500−-- stream can run in parallel. The operators start with a @|@; we can read '|$'

501−-- as "@parallel dollar@" to remember that @|@ comes before '$'.

502−--

503−-- Imports for the code snippets below:

504−--

505−-- @

506−-- import Streamly

507−-- import qualified Streamly.Prelude as S

508−-- import Control.Concurrent

509−-- @

510−

511−-- $sum

512−-- The 'Semigroup' operation '<>' of each stream type combines two streams in a

513−-- type specific manner. This section provides polymorphic versions of '<>'

514−-- which can be used to combine two streams in a predetermined way irrespective

515−-- of the type.

516−

517−-- XXX An alternative design choice would be to let a control parameter affect

518−-- the nearest SVar only and then it gets cleared. The benefit of the current

519−-- choice is that it is simply just like global configuration, just like state

520−-- behaves, so should be easy to comprehend. But it has the downside of leaking

521−-- to undesired actions, that is we can forget to reset it.

522−--

523−-- $concurrency

524−--

525−-- These combinators can be used at any point in a stream composition to set

526−-- parameters to control the concurrency of the /argument stream/. A control

527−-- parameter set at any point remains effective for any concurrent combinators

528−-- used in the argument stream until it is reset by using the combinator again.

529−-- These control parameters have no effect on non-concurrent combinators in the

530−-- stream, or on non-concurrent streams.

531−--

532−-- /Pitfall:/ Remember that 'maxBuffer' in the following example applies to

533−-- 'mapM' and any other combinators that may follow it, and it does not apply

534−-- to the combinators before it:

535−--

536−-- @

537−-- ...

538−-- $ maxBuffer 10

539−-- $ S.mapM ...

540−-- ...

541−-- @

542−--

543−-- If we use '&' instead of '$' the situation will reverse, in the following

544−-- example, 'maxBuffer' does not apply to 'mapM', it applies to combinators

545−-- that come before it, because those are the arguments to 'maxBuffer':

546−--

547−-- @

548−-- ...

549−-- & maxBuffer 10

550−-- & S.mapM ...

551−-- ...

552−-- @

553−

554−-- $adapters

555−--

556−-- You may want to use different stream composition styles at different points

557−-- in your program. Stream types can be freely converted or adapted from one

558−-- type to another. The 'IsStream' type class facilitates type conversion of

559−-- one stream type to another. It is not used directly, instead the type

560−-- combinators provided below are used for conversions.

561−--

562−-- To adapt from one monomorphic type (e.g. 'AsyncT') to another monomorphic

563−-- type (e.g. 'SerialT') use the 'adapt' combinator. To give a polymorphic code

564−-- a specific interpretation or to adapt a specific type to a polymorphic type

565−-- use the type specific combinators e.g. 'fromAsync' or 'fromWSerial'. You

566−-- cannot adapt polymorphic code to polymorphic code, as the compiler would not know

567−-- which specific type you are converting from or to. If you see a an

568−-- @ambiguous type variable@ error then most likely you are using 'adapt'

569−-- unnecessarily on polymorphic code.

570−

571−-- | Same as 'Streamly.Prelude.concatFoldableWith'

572−--

573−-- @since 0.1.0

574−{-# DEPRECATED foldWith "Please use 'Streamly.Prelude.concatFoldableWith' instead." #-}

575−{-# INLINEABLE foldWith #-}

576−foldWith :: (IsStream t, Foldable f) => (t m a -> t m a -> t m a) -> f (t m a) -> t m a

577−foldWith = P.concatFoldableWith

578−

579−-- | Same as 'Streamly.Prelude.concatMapFoldableWith'

580−--

581−-- @since 0.1.0

582−{-# DEPRECATED foldMapWith "Please use 'Streamly.Prelude.concatMapFoldableWith' instead." #-}

583−{-# INLINEABLE foldMapWith #-}

584−foldMapWith :: (IsStream t, Foldable f) => (t m b -> t m b -> t m b) -> (a -> t m b) -> f a -> t m b

585−foldMapWith = P.concatMapFoldableWith

586−

587−-- | Same as 'Streamly.Prelude.concatForFoldableWith'

588−--

589−-- @since 0.1.0

590−{-# DEPRECATED forEachWith "Please use 'Streamly.Prelude.concatForFoldableWith' instead." #-}

591−{-# INLINEABLE forEachWith #-}

592−forEachWith :: (IsStream t, Foldable f) => (t m b -> t m b -> t m b) -> f a -> (a -> t m b) -> t m b

593−forEachWith = P.concatForFoldableWith

594−

595−{-# DEPRECATED serially "Please use 'Streamly.Prelude.fromSerial' instead." #-}

596−serially :: IsStream t => SerialT m a -> t m a

597−serially = fromSerial

598−

599−{-# DEPRECATED wSerially "Please use 'Streamly.Prelude.fromWSerial' instead." #-}

600−wSerially :: IsStream t => WSerialT m a -> t m a

601−wSerially = fromWSerial

602−

603−{-# DEPRECATED asyncly "Please use 'Streamly.Prelude.fromAsync' instead." #-}

604−asyncly :: IsStream t => AsyncT m a -> t m a

605−asyncly = fromAsync

606−

607−{-# DEPRECATED aheadly "Please use 'Streamly.Prelude.fromAhead' instead." #-}

608−aheadly :: IsStream t => AheadT m a -> t m a

609−aheadly = fromAhead

610−

611−{-# DEPRECATED wAsyncly "Please use 'Streamly.Prelude.fromWAsync' instead." #-}

612−wAsyncly :: IsStream t => WAsyncT m a -> t m a

613−wAsyncly = fromWAsync

614−

615−{-# DEPRECATED parallely "Please use 'Streamly.Prelude.fromParallel' instead." #-}

616−parallely :: IsStream t => ParallelT m a -> t m a

617−parallely = fromParallel

618−

619−{-# DEPRECATED zipSerially "Please use 'Streamly.Prelude.fromZipSerial' instead." #-}

620−zipSerially :: IsStream t => ZipSerialM m a -> t m a

621−zipSerially = fromZipSerial

622−

623−{-# DEPRECATED zipAsyncly "Please use 'Streamly.Prelude.fromZipAsync' instead." #-}

624−zipAsyncly :: IsStream t => ZipAsyncM m a -> t m a

625−zipAsyncly = fromZipAsync

src/Streamly/Data/Fold/Prelude.hs+31−13

@@ -6,8 +6,27 @@

66-- Stability : released

77-- Portability : GHC

88--

9+-- All Fold related combinators including the streamly-core

10+-- "Streamly.Data.Fold" module, concurrency, unordered container operations.

11+--

912module Streamly.Data.Fold.Prelude

10− ( module Streamly.Data.Fold

13+ (

14+ -- * "Streamly.Data.Fold"

15+ -- | All "Streamly.Data.Fold" combinators are re-exported via this

16+ -- module. For more pre-release combinators also see

17+ -- "Streamly.Internal.Data.Fold" module.

18+ module Streamly.Data.Fold

19+ -- * Concurrent Operations

20+ -- ** Configuration

21+ , Config

22+ , maxBuffer

23+ , boundThreads

24+ , inspect

25+

26+ -- ** Combinators

27+ , parEval

28+

29+ -- * Container Related

1130 , toHashMapIO

1231 )

1332where

@@ -17,27 +36,26 @@ import Data.HashMap.Strict (HashMap)

1736import Data.Hashable (Hashable)

1837import Streamly.Data.Fold

1938import Streamly.Internal.Data.Fold (toContainerIO)

39+import Streamly.Internal.Data.Fold.Concurrent

2040import Streamly.Internal.Data.IsMap.HashMap ()

2141

2242-- | Split the input stream based on a hashable component of the key field and

2343-- fold each split using the given fold. Useful for map/reduce, bucketizing

2444-- the input in different bins or for generating histograms.

2545--

26−-- Example:

27−--

2846-- >>> import Data.HashMap.Strict (HashMap, fromList)

2947-- >>> import qualified Streamly.Data.Fold.Prelude as Fold

3048-- >>> import qualified Streamly.Data.Stream as Stream

31−-- >>> import Streamly.Data.Fold.Prelude (toHashMapIO)

32−-- >>> :{

33−-- do

34−-- let input = Stream.fromList [("ONE",1),("ONE",1.1),("TWO",2), ("TWO",2.2)]

35−-- classify = toHashMapIO fst (Fold.lmap snd Fold.toList)

36−-- x <- Stream.fold classify input :: IO (HashMap String [Double])

37−-- let y = fromList [("ONE",[1.0,1.1]),("TWO",[2.0,2.2])]

38−-- return (x == y)

39−-- :}

40−-- True

49+--

50+-- Consider a stream of key value pairs:

51+--

52+-- >>> input = Stream.fromList [("k1",1),("k1",1.1),("k2",2), ("k2",2.2)]

53+--

54+-- Classify each key to a different hash bin and fold the bins:

55+--

56+-- >>> classify = Fold.toHashMapIO fst (Fold.lmap snd Fold.toList)

57+-- >>> Stream.fold classify input :: IO (HashMap String [Double])

58+-- fromList [("k2",[2.0,2.2]),("k1",[1.0,1.1])]

4159--

4260-- /Pre-release/

4361--

src/Streamly/Data/Fold/Tee.hs+1−1

@@ -13,4 +13,4 @@ module Streamly.Data.Fold.Tee

1313 )

1414where

1515

16−import Streamly.Internal.Data.Fold.Tee

16+import Streamly.Internal.Data.Fold

src/Streamly/Data/Stream/Concurrent.hsdeleted−194

@@ -1,194 +0,0 @@

1−-- |

2−-- Module : Streamly.Data.Stream.Concurrent

4−-- License : BSD-3-Clause

5−-- Maintainer : streamly@composewell.com

6−-- Stability : released

7−-- Portability : GHC

8−--

9−-- This module provides concurrent streaming abstractions.

10−--

11−-- == Programming Tips

12−--

13−-- The names in this module do not conflict with other stream modules,

14−-- therefore, you can import it in the same namespace:

15−--

16−-- >>> import qualified Streamly.Data.Stream.Concurrent as Stream

17−

18−module Streamly.Data.Stream.Concurrent

19− (

20− -- * Concurrency

21− -- $concurrency

22− MonadAsync

23−

24− -- ** Configuration

25− , Config

26−

27− -- *** Limits

28− , maxThreads

29− , maxBuffer

30−

31− -- *** Rate Control

32− , Rate(..)

33− , rate

34− , avgRate

35− , minRate

36− , maxRate

37− , constRate

38−

39− -- *** Stop behavior

40− , StopWhen (..)

41− , stopWhen

42−

43− -- *** Scheduling behavior

44− , eager

45− , ordered

46− , interleaved

47−

48− -- *** Diagnostics

49− , inspect

50−

51− -- ** Combinators

52− -- | Stream combinators using a concurrent channel.

53−

54− -- *** Evaluate

55− -- | Evaluates a serial stream asynchronously using a concurency channel.

56−

57− , parEval

58−

59− -- *** Generate

60− -- | Uses a concurrency channel to evaluate multiple actions concurrently.

61− , parRepeatM

62− , parReplicateM

63− , fromCallback

64−

65− -- *** Map

66− -- | Uses a concurrency channel to evaluate multiple mapped actions

67− -- concurrently.

68−

69− , parMapM

70− , parSequence

71−

72− -- *** Combine two

73− -- | Use a channel for each pair.

74− , parZipWithM

75− , parZipWith

76− , parMergeByM

77− , parMergeBy

78−

79− -- *** List of streams

80− -- | Shares a single channel across many streams.

81−

82− , parList

83− -- , zipWithM

84− -- , zipWith

85−

86− -- *** Stream of streams

87− -- **** Apply

88−

89− , parApply

90−

91− -- **** Concat

92− -- | Shares a single channel across many streams.

93−

94− , parConcat

95− , parConcatMap

96−

97− -- **** ConcatIterate

98− , parConcatIterate

99−

100− -- *** Observation

101− , parTapCount

102−

103− -- ** Deprecated

104− , tapCount

105− )

106−where

107−

108−import Streamly.Internal.Data.Stream.Concurrent

109−import Prelude hiding (mapM, sequence, concat, concatMap, zipWith)

110−

111−-- $concurrency

112−--

113−-- == Concurrency Channels

114−--

115−-- At a lower level, concurrency is implemented using channels that support

116−-- concurrent evaluation of streams. We create a channel, and add one or more

117−-- streams to it. The channel evaluates multiple streams concurrently and then

118−-- generates a single output stream from the results. How the streams are

119−-- combined depends on the configuration of the channel.

120−--

121−-- == Concurrency Primitives

122−--

123−-- There are only a few fundamental abstractions for concurrency, 'parEval',

124−-- 'parConcatMap', and 'parConcatIterate', all concurrency combinators can be

125−-- expressed in terms of these.

126−--

127−-- 'parEval', evaluates a single stream asynchronously, a worker thread runs

128−-- the stream and buffers the results, and the consumer of the stream runs in

129−-- another thread consuming it from the buffer, thus decoupling the production

130−-- and consumption of the stream. This can be used to run different stages of a

131−-- pipeline concurrently.

132−--

133−-- 'parConcatMap' is used to evaluate multiple streams concurrently and combine

134−-- the results. A stream generator function is mapped to the input stream and

135−-- all the generated streams are then evaluated concurrently, and the results

136−-- are combined.

137−--

138−-- 'parConcatIterate' is like 'parConcatMap' but iterates a stream generator

139−-- function recursively over the stream. This can be used to traverse trees or

140−-- graphs.

141−--

142−-- == Concurrency Configuration

143−--

144−-- Concurrent combinators take a 'Config' argument which controls the

145−-- concurrent behavior. For example, maximum number of threads to be used

146−-- ('maxThreads') or the maxmimum size of the buffer ('maxBuffer'), or how the

147−-- streams are scheduled with respect to each other ('interleaved'), or how the

148−-- results are consumed ('ordered').

149−--

150−-- Configuration is specified as @Config -> Config@ modifier functions that can

151−-- be composed together using function composition. For example, to specify the

152−-- maximum threads we can use @parConcatMap (maxThreads 10)@ if we also want to

153−-- specify the maximum buffer we can compose the two options @parConcatMap

154−-- (maxThreads 10 . maxBuffer 100)@. To use default configuration use 'id' as

155−-- the config modifier e.g. @parConcatMap id@.

156−--

157−-- See the @Configuration@ section and individual configuration options'

158−-- documentation for the default behavior and default values of configuration

159−-- parameters.

160−--

161−-- == Scheduling behavior

162−--

163−-- The most important configuration option is to control whether the output of

164−-- the concurrent execution is consumed in the same order as the corresponding

165−-- actions in the input stream or as soon as they arrive. The default is the

166−-- latter, however, we can enforce the original order by using the 'ordered'

167−-- option.

168−--

169−-- Another important option controls whether the number of worker threads are

170−-- automatically increased and decreased based on the consumption rate or

171−-- threads are started as aggresively as possible until the 'maxThreads' or

172−-- 'maxBuffer' limits are hit. The default is the former. However, the 'eager'

173−-- option can be enabled to use the latter behavior. When 'eager' is on, even

174−-- if the stream consumer thread blocks it does not make any impact on the

175−-- scheduling of the available tasks.

176−--

177−-- == Concurrent Combinators

178−--

179−-- Using the few fundamental concurrency primitives we can implement all the

180−-- usual streaming combinators with concurrent behavior. Combinators like

181−-- 'unfoldrM', 'iterateM' that are inherently serial can be evaluated

182−-- concurrently with respect to the consumer pipeline using 'parEval'.

183−-- Combinators like 'zipWithM', 'mergeByM' can also use 'parEval' on the input

184−-- streams to evaluate them concurrently before combining.

185−--

186−-- Combinators like 'repeatM', 'replicateM', 'fromListM', 'sequence', 'mapM' in

187−-- which all actions are independent of each other can be made concurrent using

188−-- the 'parConcatMap' operation.

189−--

190−-- A concurrent 'repeatM' repeats an action using multiple concurrent

191−-- executions of the action. Similarly, a concurrent 'mapM' performs the mapped

192−-- action in independent threads.

193−--

194−-- Some common concurrent combinators are provided in this module.

src/Streamly/Data/Stream/Exception.hsdeleted−25

@@ -1,25 +0,0 @@

1−-- |

2−-- Module : Streamly.Data.Stream.Exception

4−-- License : BSD-3-Clause

5−-- Maintainer : streamly@composewell.com

6−-- Stability : released

7−-- Portability : GHC

8−--

9−-- This module is designed such that it does not conflict with

10−-- "Streamly.Data.Stream" module.

11−--

12−-- >>> import qualified Streamly.Data.Stream.Exception as Stream

13−--

14−module Streamly.Data.Stream.Exception

15− (

16− -- * Lifted Exceptions

17− after

18− , bracket

19− -- , bracket3

20− , finally

21− )

22−where

23−

24−import Streamly.Internal.Data.Stream.Exception.Lifted

25− ( bracket, finally, after )

src/Streamly/Data/Stream/MkType.hs+41−10

@@ -7,7 +7,8 @@

77-- Portability : GHC

88--

99-- Template Haskell macros to create custom newtype wrappers for the 'Stream'

10−-- type, deriving all the usual instances.

10+-- type. See the examples below to create the standard stream types that were

11+-- available in streamly versions before 0.9.0.

1112--

1213-- To use this module, the following extensions must be enabled:

1314--

@@ -23,31 +24,61 @@

2324-- >>> import Streamly.Data.Stream.MkType

2425-- >>> import qualified Streamly.Data.Stream.Prelude as Stream

2526--

26−-- Example, create an applicative type with zipping apply:

27+-- For 'Streamly.Prelude.AsyncT' monad type with a concurrent cross product

28+-- bind:

2729--

2830-- >>> :{

29−-- zipApply = Stream.zipWith ($)

30−-- $(mkZipType "ZipStream" "zipApply" False)

31+-- bind = flip (Stream.parConcatMap id)

32+-- $(mkCrossType "AsyncT" "bind" True)

3133-- :}

3234--

33−-- Example, create an applicative type with concurrent zipping apply:

35+-- For 'Streamly.Prelude.WAsyncT' monad type with a concurrent interleaved

36+-- bind:

3437--

3538-- >>> :{

36−-- parApply = Stream.parApply id

37−-- $(mkZipType "ParZipStream" "parApply" True)

39+-- bind = flip (Stream.parConcatMap (Stream.interleaved True))

40+-- $(mkCrossType "WAsyncT" "bind" True)

3841-- :}

3942--

40−-- Example, create a monad type with an eager concurrent cross product bind:

43+-- For 'Streamly.Prelude.AheadT' monad type with a concurrent ordered

44+-- cross product bind:

45+--

46+-- >>> :{

47+-- bind = flip (Stream.parConcatMap (Stream.ordered True))

48+-- $(mkCrossType "AheadT" "bind" True)

49+-- :}

50+--

51+-- For 'Streamly.Prelude.ParallelT' monad type with an eager concurrent cross

52+-- product bind:

4153--

4254-- >>> :{

4355-- parBind = flip (Stream.parConcatMap (Stream.eager True))

44−-- $(mkCrossType "ParEagerStream" "parBind" True)

56+-- $(mkCrossType "ParallelT" "parBind" True)

57+-- :}

58+--

59+-- For 'Streamly.Prelude.ZipSerialM' serial zipping applicative type:

60+--

61+-- >>> :{

62+-- zipApply = Stream.zipWith ($)

63+-- $(mkZipType "ZipSerialM" "zipApply" False)

64+-- :}

65+--

66+-- For 'Streamly.Prelude.ZipAsync' concurrent zipping applicative type:

67+--

68+-- >>> :{

69+-- parApply = Stream.parApply id

70+-- $(mkZipType "ZipAsync" "parApply" True)

4571-- :}

4672--

4773-- Instead of using these macros directly you could use the generated code as

4874-- well. Use these macros in ghci to generate the required code and paste it in

4975-- your package, you can customize the code as desired. See the docs of the

50−-- macros below for examples about how to view the generated code.

76+-- macros below for examples about how to view the generated code. For example:

77+--

78+-- >>> bind = flip (Stream.parConcatMap id)

79+-- >>> expr <- runQ (mkCrossType "AsyncT" "bind" True)

80+--

81+-- >> putStrLn $ pprint expr

5182--

5283module Streamly.Data.Stream.MkType

5384 (

src/Streamly/Data/Stream/Prelude.hs+217−15

@@ -6,25 +6,227 @@

66-- Stability : experimental

77-- Portability : GHC

88--

9−-- This module re-exports the "Streamly.Data.Stream" module from the

10−-- "streamly-core" package and additionally provides concurrency, time and

11−-- lifted exception operations as well in a single module.

9+-- For upgrading to streamly-0.9.0+ please read the

10+-- <https://github.com/composewell/streamly/blob/streamly-0.10.0/docs/User/Project/Upgrading-0.8-to-0.9.md Streamly-0.9.0 upgrade guide>.

11+-- Also, see the "Streamly.Data.Stream.MkType" module for direct replacement of

12+-- stream types that have been removed in 0.9.0.

1213--

13−-- Also see the following modules for more pre-release operations:

14−--

15−-- * "Streamly.Internal.Data.Stream.Concurrent"

16−-- * "Streamly.Internal.Data.Stream.Time"

17−-- * "Streamly.Internal.Data.Stream.Exception.Lifted"

14+-- All Stream related combinators including the streamly-core

15+-- "Streamly.Data.Stream" module, concurrency, time and lifted

16+-- exception operations. For more pre-release operations also see

17+-- "Streamly.Internal.Data.Stream.Prelude" module.

1818--

1919module Streamly.Data.Stream.Prelude

20− ( module Streamly.Data.Stream

21− , module Streamly.Data.Stream.Concurrent

22− , module Streamly.Data.Stream.Time

23− , module Streamly.Data.Stream.Exception

20+ (

21+ -- * "Streamly.Data.Stream"

22+ -- | All "Streamly.Data.Stream" combinators are re-exported via this

23+ -- module. For more pre-release combinators also see

24+ -- "Streamly.Internal.Data.Stream" module.

25+ module Streamly.Data.Stream

26+

27+ -- * Concurrent Operations

28+ -- $concurrency

29+

30+ -- ** Types

31+ , MonadAsync

32+

33+ -- ** Configuration

34+ , Config

35+

36+ -- *** Limits

37+ , maxThreads

38+ , maxBuffer

39+

40+ -- *** Rate Control

41+ , Rate(..)

42+ , rate

43+ , avgRate

44+ , minRate

45+ , maxRate

46+ , constRate

47+

48+ -- *** Stop behavior

49+ , StopWhen (..)

50+ , stopWhen

51+

52+ -- *** Scheduling behavior

53+ , eager

54+ , ordered

55+ , interleaved

56+

57+ -- *** Diagnostics

58+ , inspect

59+

60+ -- ** Combinators

61+ -- | Stream combinators using a concurrent channel.

62+

63+ -- *** Evaluate

64+ -- | Evaluate a stream as a whole concurrently with respect to the consumer

65+ -- of the stream.

66+

67+ , parEval

68+

69+ -- *** Generate

70+ -- | Generate a stream by evaluating multiple actions concurrently.

71+ , parRepeatM

72+ , parReplicateM

73+ , fromCallback

74+

75+ -- *** Map

76+ -- | Map actions on a stream such that the mapped actions are evaluated

77+ -- concurrently with each other.

78+

79+ , parMapM

80+ , parSequence

81+

82+ -- *** Combine two

83+ -- | Combine two streams such that each stream as a whole is evaluated

84+ -- concurrently with respect to the other stream as well as the consumer of

85+ -- the resulting stream.

86+ , parZipWithM

87+ , parZipWith

88+ , parMergeByM

89+ , parMergeBy

90+

91+ -- *** List of streams

92+ -- | Shares a single channel across many streams.

93+

94+ , parList

95+ -- , zipWithM

96+ -- , zipWith

97+

98+ -- *** Stream of streams

99+ -- **** Apply

100+

101+ , parApply

102+

103+ -- **** Concat

104+ -- | Shares a single channel across many streams.

105+

106+ , parConcat

107+ , parConcatMap

108+

109+ -- **** ConcatIterate

110+ , parConcatIterate

111+

112+ -- *** Observation

113+ , parTapCount

114+

115+ -- * Time Related

116+

117+ -- ** Timers

118+ , interject

119+

120+ -- ** Trimming

121+ , takeInterval

122+ , dropInterval

123+

124+ -- ** Chunking

125+ , intervalsOf

126+

127+ -- ** Sampling

128+ , sampleIntervalEnd

129+ , sampleIntervalStart

130+ , sampleBurstEnd

131+ , sampleBurstStart

132+

133+ -- * Lifted Exceptions

134+ , after

135+ , bracket

136+ -- , bracket3

137+ , finally

138+

139+ -- ** Deprecated

140+ , tapCount

24141 )

25142where

26143

27144import Streamly.Data.Stream

28−import Streamly.Data.Stream.Concurrent

29−import Streamly.Data.Stream.Exception

30−import Streamly.Data.Stream.Time

145+import Streamly.Internal.Data.Stream.Prelude

146+

147+-- $concurrency

148+--

149+-- == Channels

150+--

151+-- At a lower level, concurrency is implemented using channels that support

152+-- concurrent evaluation of streams. We create a channel, and add one or more

153+-- streams to it. The channel evaluates multiple streams concurrently and then

154+-- generates a single output stream from the results. How the streams are

155+-- combined depends on the configuration of the channel.

156+--

157+-- == Primitives

158+--

159+-- There are only a few fundamental abstractions for concurrency, 'parEval',

160+-- 'parConcatMap', and 'parConcatIterate', all concurrency combinators can be

161+-- expressed in terms of these.

162+--

163+-- 'parEval' evaluates a stream as a whole asynchronously with respect to

164+-- the consumer of the stream. A worker thread evaluates multiple elements of

165+-- the stream ahead of time and buffers the results; the consumer of the stream

166+-- runs in another thread consuming the elements from the buffer, thus

167+-- decoupling the production and consumption of the stream. 'parEval' can be

168+-- used to run different stages of a pipeline concurrently.

169+--

170+-- 'parConcatMap' is used to evaluate multiple actions in a stream concurrently

171+-- with respect to each other or to evaluate multiple streams concurrently and

172+-- combine the results. A stream generator function is mapped to the input

173+-- stream and all the generated streams are then evaluated concurrently, and

174+-- the results are combined.

175+--

176+-- 'parConcatIterate' is like 'parConcatMap' but iterates a stream generator

177+-- function recursively over the stream. This can be used to traverse trees or

178+-- graphs.

179+--

180+-- == Configuration

181+--

182+-- Concurrent combinators take a 'Config' argument which controls the

183+-- concurrent behavior. For example, maximum number of threads to be used

184+-- ('maxThreads') or the maxmimum size of the buffer ('maxBuffer'), or how the

185+-- streams are scheduled with respect to each other ('interleaved'), or how the

186+-- results are consumed ('ordered').

187+--

188+-- Configuration is specified as @Config -> Config@ modifier functions that can

189+-- be composed together using function composition. For example, to specify the

190+-- maximum threads we can use @parConcatMap (maxThreads 10)@ if we also want to

191+-- specify the maximum buffer we can compose the two options @parConcatMap

192+-- (maxThreads 10 . maxBuffer 100)@. To use default configuration use 'id' as

193+-- the config modifier e.g. @parConcatMap id@.

194+--

195+-- See the @Configuration@ section and individual configuration options'

196+-- documentation for the default behavior and default values of configuration

197+-- parameters.

198+--

199+-- == Scheduling

200+--

201+-- The most important configuration option is to control whether the output of

202+-- the concurrent execution is consumed in the same order as the corresponding

203+-- actions in the input stream or as soon as they arrive. The default is the

204+-- latter, however, we can enforce the original order by using the 'ordered'

205+-- option.

206+--

207+-- Another important option controls whether the number of worker threads are

208+-- automatically increased and decreased based on the consumption rate or

209+-- threads are started as aggresively as possible until the 'maxThreads' or

210+-- 'maxBuffer' limits are hit. The default is the former. However, the 'eager'

211+-- option can be enabled to use the latter behavior. When 'eager' is on, even

212+-- if the stream consumer thread blocks it does not make any impact on the

213+-- scheduling of the available tasks.

214+--

215+-- == Combinators

216+--

217+-- Using the few fundamental concurrency primitives we can implement all the

218+-- usual streaming combinators with concurrent behavior. Combinators like

219+-- 'unfoldrM', 'iterateM' that are inherently serial can be evaluated

220+-- concurrently with respect to the consumer pipeline using 'parEval'.

221+-- Combinators like 'zipWithM', 'mergeByM' can also use 'parEval' on the input

222+-- streams to evaluate them concurrently before combining.

223+--

224+-- Combinators like 'repeatM', 'replicateM', 'fromListM', 'sequence', 'mapM' in

225+-- which all actions are independent of each other can be made concurrent using

226+-- the 'parConcatMap' operation.

227+--

228+-- A concurrent 'repeatM' repeats an action using multiple concurrent

229+-- executions of the action. Similarly, a concurrent 'mapM' performs the mapped

230+-- action in independent threads.

231+--

232+-- Some common concurrent combinators are provided in this module.

src/Streamly/Data/Stream/Time.hsdeleted−37

@@ -1,37 +0,0 @@

1−-- |

2−-- Module : Streamly.Data.Stream.Time

4−-- License : BSD-3-Clause

5−-- Maintainer : streamly@composewell.com

6−-- Stability : released

7−-- Portability : GHC

8−--

9−-- This module provides time related streaming abstractions.

10−--

11−-- The names in this module do not conflict with other stream modules,

12−-- therefore, you can import it in the same namespace:

13−--

14−-- >>> import qualified Streamly.Data.Stream.Time as Stream

15−--

16−module Streamly.Data.Stream.Time

17− (

18− -- * Time

19− -- ** Timers

20− interject

21−

22− -- ** Trimming

23− , takeInterval

24− , dropInterval

25−

26− -- ** Chunking

27− , intervalsOf

28−

29− -- ** Sampling

30− , sampleIntervalEnd

31− , sampleIntervalStart

32− , sampleBurstEnd

33− , sampleBurstStart

34− )

35−where

36−

37−import Streamly.Internal.Data.Stream.Time

src/Streamly/Data/Unicode/Stream.hsdeleted−99

@@ -1,99 +0,0 @@

1−-- |

2−-- Module : Streamly.Data.Unicode.Stream

4−--

5−-- License : BSD3

6−-- Maintainer : streamly@composewell.com

7−-- Stability : experimental

8−-- Portability : GHC

9−--

10−-- = Processing Unicode Strings

11−--

12−-- A 'Char' stream is the canonical representation to process Unicode strings.

13−-- It can be processed efficiently using regular stream processing operations.

14−-- A byte stream of Unicode text read from an IO device or from an

15−-- 'Streamly.Data.Array.Foreign.Array' in memory can be decoded into a 'Char' stream

16−-- using the decoding routines in this module. A 'String' (@[Char]@) can be

17−-- converted into a 'Char' stream using 'Streamly.Prelude.fromList'. An @Array

18−-- Char@ can be 'Streamly.Prelude.unfold'ed into a stream using the array

19−-- 'Streamly.Data.Array.Foreign.read' unfold.

20−--

21−-- = Storing Unicode Strings

22−--

23−-- A stream of 'Char' can be encoded into a byte stream using the encoding

24−-- routines in this module and then written to IO devices or to arrays in

25−-- memory.

26−--

27−-- If you have to store a 'Char' stream in memory you can convert it into a

28−-- 'String' using 'Streamly.Prelude.toList' or using the

29−-- 'Streamly.Data.Fold.toList' fold. The 'String' type can be more efficient

30−-- than pinned arrays for short and short lived strings.

31−--

32−-- For longer or long lived streams you can 'Streamly.Prelude.fold' the 'Char'

33−-- stream as @Array Char@ using the array 'Streamly.Data.Array.Foreign.write' fold.

34−-- The 'Array' type provides a more compact representation and pinned memory

35−-- reducing GC overhead. If space efficiency is a concern you can use

36−-- 'encodeUtf8'' on the 'Char' stream before writing it to an 'Array' providing

37−-- an even more compact representation.

38−--

39−-- = String Literals

40−--

41−-- @Stream Identity Char@ and @Array Char@ are instances of 'IsString' and

42−-- 'IsList', therefore, 'OverloadedStrings' and 'OverloadedLists' extensions

43−-- can be used for convenience when specifying unicode strings literals using

44−-- these types.

45−--

46−-- = Pitfalls

47−--

48−-- * Case conversion: Some unicode characters translate to more than one code

49−-- point on case conversion. The 'toUpper' and 'toLower' functions in @base@

50−-- package do not handle such characters. Therefore, operations like @map

51−-- toUpper@ on a character stream or character array may not always perform

52−-- correct conversion.

53−-- * String comparison: In some cases, visually identical strings may have

54−-- different unicode representations, therefore, a character stream or

55−-- character array cannot be directly compared. A normalized comparison may be

56−-- needed to check string equivalence correctly.

57−--

58−-- = Experimental APIs

59−--

60−-- Some experimental APIs to conveniently process text using the

61−-- @Array Char@ represenation directly can be found in

62−-- "Streamly.Internal.Memory.Unicode.Array".

63−

64−-- XXX an unpinned array representation can be useful to store short and short

65−-- lived strings in memory.

66−--

67−module Streamly.Data.Unicode.Stream

68− {-# DEPRECATED "Please use Streamly.Unicode.Stream module from the streamly-core package." #-}

69− (

70− -- * Construction (Decoding)

71− decodeLatin1

72− , decodeUtf8

73−

74− -- * Elimination (Encoding)

75− , encodeLatin1

76− , encodeUtf8

77− {-

78− -- * Operations on character strings

79− , strip -- (dropAround isSpace)

80− , stripEnd

81− , stripStart

82− -}

83− -- Not exposing these yet as we have consider these with respect to Unicode

84− -- segmentation routines which are yet to be implemented.

85− -- -- * Transformation

86− -- , lines

87− -- , words

88− -- , unlines

89− -- , unwords

90−

91− -- * Deprecations

92− , decodeUtf8Lax

93− , encodeLatin1Lax

94− , encodeUtf8Lax

95− )

96−where

97−

98−import Streamly.Internal.Unicode.Stream

99−import Prelude hiding (lines, words, unlines, unwords)

src/Streamly/Internal/Data/Array/ArrayMacros.hdeleted−40

@@ -1,40 +0,0 @@

1−-------------------------------------------------------------------------------

2−-- Macros to access array pointers

3−-------------------------------------------------------------------------------

4−

5−-- The Storable instance of () has size 0. We ensure that the size is non-zero

6−-- to avoid a zero sized element and issues due to that.

7−-- See https://mail.haskell.org/pipermail/libraries/2022-January/thread.html

8−--

9−-- XXX Check the core to see if max can be statically eliminated. llvm can

10−-- eliminate the comparison, but not sure if GHC NCG can.

11−#define SIZE_OF(a) max 1 (sizeOf (Proxy :: Proxy a))

12−#define STORABLE_SIZE_OF(a) max 1 (sizeOf (undefined :: a))

13−

14−-- Move the pointer to ith element of specified type. Type is specified as the

15−-- type variable in the signature of the function where this macro is used.

16−#define PTR_NEXT(ptr,a) ptr `plusPtr` STORABLE_SIZE_OF(a)

17−#define PTR_PREV(ptr,a) ptr `plusPtr` negate (STORABLE_SIZE_OF(a))

18−

19−#define PTR_INDEX(ptr,i,a) ptr `plusPtr` (STORABLE_SIZE_OF(a) * i)

20−#define PTR_RINDEX(ptr,i,a) ptr `plusPtr` negate (STORABLE_SIZE_OF(a) * (i + 1))

21−

22−-- XXX If we know that the array is guaranteed to have size multiples of the

23−-- element size then we can use a simpler check saying "ptr < end". Since we

24−-- always allocate in multiples of elem we can use the simpler check and assert

25−-- the rigorous check.

26−#define PTR_VALID(ptr,end,a) ptr `plusPtr` STORABLE_SIZE_OF(a) <= end

27−#define PTR_INVALID(ptr,end,a) ptr `plusPtr` STORABLE_SIZE_OF(a) > end

28−

29−-------------------------------------------------------------------------------

30−-- Macros to access array indices

31−-------------------------------------------------------------------------------

32−

33−#define INDEX_NEXT(i,a) i + SIZE_OF(a)

34−#define INDEX_PREV(i,a) i - SIZE_OF(a)

35−

36−#define INDEX_OF(base,i,a) base + (SIZE_OF(a) * i)

37−#define RINDEX_OF(base,i,a) base - (SIZE_OF(a) * (i + 1))

38−

39−#define INDEX_VALID(i,end,a) i + SIZE_OF(a) <= end

40−#define INDEX_INVALID(i,end,a) i + SIZE_OF(a) > end

src/Streamly/Internal/Data/Channel.hsnew file+19

@@ -0,0 +1,19 @@

1+-- |

2+-- Module : Streamly.Internal.Data.Channel

4+-- License : BSD-3-Clause

5+-- Maintainer : streamly@composewell.com

6+-- Stability : experimental

7+-- Portability : GHC

9+module Streamly.Internal.Data.Channel

10+ (

11+ module Streamly.Internal.Data.Channel.Dispatcher

12+ , module Streamly.Internal.Data.Channel.Types

13+ , module Streamly.Internal.Data.Channel.Worker

14+ )

15+where

16+

17+import Streamly.Internal.Data.Channel.Dispatcher

18+import Streamly.Internal.Data.Channel.Types

19+import Streamly.Internal.Data.Channel.Worker

src/Streamly/Internal/Data/Stream/Channel/Dispatcher.hs → src/Streamly/Internal/Data/Channel/Dispatcher.hsrenamed+3−3

@@ -1,5 +1,5 @@

11-- |

2−-- Module : Streamly.Internal.Data.Stream.Channel.Dispatcher

2+-- Module : Streamly.Internal.Data.Channel.Dispatcher

44-- License : BSD-3-Clause

55-- Maintainer : streamly@composewell.com

@@ -7,7 +7,7 @@

77-- Portability : GHC

88--

99--

10−module Streamly.Internal.Data.Stream.Channel.Dispatcher

10+module Streamly.Internal.Data.Channel.Dispatcher

1111 (

1212 -- * Latency collection

1313 minThreadDelay

@@ -40,7 +40,7 @@ import Streamly.Internal.Data.Time.Units

4040

4141import qualified Data.Set as S

4242

43−import Streamly.Internal.Data.Stream.Channel.Types

43+import Streamly.Internal.Data.Channel.Types

4444

4545-------------------------------------------------------------------------------

4646-- Worker latency data processing

src/Streamly/Internal/Data/Stream/Channel/Types.hs → src/Streamly/Internal/Data/Channel/Types.hsrenamed+12−37

@@ -1,5 +1,5 @@

11-- |

2−-- Module : Streamly.Internal.Data.Stream.Channel.Types

2+-- Module : Streamly.Internal.Data.Channel.Types

44-- License : BSD-3-Clause

55-- Maintainer : streamly@composewell.com

@@ -31,7 +31,7 @@

3131-- consumer is not pulling fast enough. The number of workers may even get

3232-- reduced depending on the consuming pace.

3333--

34−module Streamly.Internal.Data.Stream.Channel.Types

34+module Streamly.Internal.Data.Channel.Types

3535 (

3636 -- * Types

3737 Count (..)

@@ -76,7 +76,7 @@ module Streamly.Internal.Data.Stream.Channel.Types

7676 , stopWhen

7777 , ordered

7878 , interleaved

79− , bound

79+ , boundThreads

8080

8181 , rate

8282 , avgRate

@@ -111,8 +111,6 @@ module Streamly.Internal.Data.Stream.Channel.Types

111111

112112 , withDiagMVar

113113 , printSVar

114−

115− , concatMapDivK

116114 )

117115where

118116

@@ -127,13 +125,11 @@ import Data.IORef (IORef, newIORef, readIORef, writeIORef)

127125import Data.Set (Set)

128126import Streamly.Internal.Data.Atomics

129127 (atomicModifyIORefCAS, atomicModifyIORefCAS_, storeLoadBarrier)

130−import Streamly.Internal.Data.SVar.Type (adaptState)

131128import Streamly.Internal.Data.Time.Clock (Clock(Monotonic), getTime)

132129import Streamly.Internal.Data.Time.Units (AbsTime, NanoSecond64(..))

133130import System.IO (hPutStrLn, stderr)

134131

135132import qualified Data.Set as Set

136−import qualified Streamly.Internal.Data.StreamK as K

137133

138134------------------------------------------------------------------------------

139135-- Common types

@@ -499,10 +495,10 @@ getInspectMode = _inspect

499495

500496-- | By default, processing of output from the worker threads is given priority

501497-- over dispatching new workers. More workers are dispatched only when there is

502−-- no output to process. With 'eager' on workers are dispatched aggresively as

503−-- long as there is more work to do irrespective of whether there is output

504−-- pending to be processed. However, dispatching may stop if 'maxThreads' or

505−-- 'maxBuffer' is reached.

498+-- no output to process. When 'eager' is set to 'True', workers are dispatched

499+-- aggresively as long as there is more work to do irrespective of whether

500+-- there is output pending to be processed by the stream consumer. However,

501+-- dispatching may stop if 'maxThreads' or 'maxBuffer' is reached.

506502--

507503-- /Note:/ This option has no effect when rate has been specified.

508504--

@@ -546,13 +542,14 @@ interleaved flag st = st { _interleaved = flag }

546542getInterleaved :: Config -> Bool

547543getInterleaved = _interleaved

548544

545+-- TODO: Make it consistently take effect everywhere.

546+

549547-- | Spawn bound threads (i.e., spawn threads using 'forkOS' instead of

550548-- 'forkIO'). The default value is 'False'.

551549--

552−-- Currently, this only takes effect for concurrent folds. TODO: Make it

553−-- consistently take effect everywhere.

554−bound :: Bool -> Config -> Config

555−bound flag st = st { _bound = flag }

550+-- Currently, this only takes effect only for concurrent folds.

551+boundThreads :: Bool -> Config -> Config

552+boundThreads flag st = st { _bound = flag }

556553

557554getBound :: Config -> Bool

558555getBound = _bound

@@ -818,25 +815,3 @@ cleanupSVar workerSet = do

818815 Prelude.mapM_ (`throwTo` ThreadAbort)

819816 -- (Prelude.filter (/= self) $ Set.toList workers)

820817 (Set.toList workers)

821−

822−-------------------------------------------------------------------------------

823−-- Evaluator

824−-------------------------------------------------------------------------------

825−

826−-- | @concatMapDivK useTail useHead stream@, divides the stream in head and

827−-- tail, maps a stream generator on the head and maps an action on the tail of

828−-- a stream. Returns the stream generated by the head.

829−--

830−-- Used for concurrent evaluation of streams using a Channel.

831−{-# INLINE concatMapDivK #-}

832−concatMapDivK :: Monad m =>

833− (K.StreamK m a -> m ())

834− -> (a -> K.StreamK m b)

835− -> K.StreamK m a

836− -> K.StreamK m b

837−concatMapDivK useTail useHead stream =

838− K.mkStream $ \st yld sng stp -> do

839− let foldShared = K.foldStreamShared st yld sng stp

840− single a = foldShared $ useHead a

841− yieldk a r = useTail r >> single a

842− in K.foldStreamShared (adaptState st) yieldk single stp stream

src/Streamly/Internal/Data/Stream/Channel/Worker.hs → src/Streamly/Internal/Data/Channel/Worker.hsrenamed+3−3

@@ -1,5 +1,5 @@

11-- |

2−-- Module : Streamly.Internal.Data.Stream.Concurrent.Channel.Worker

2+-- Module : Streamly.Internal.Data.Channel.Worker

44-- License : BSD-3-Clause

55-- Maintainer : streamly@composewell.com

@@ -8,7 +8,7 @@

88--

99-- Collecting results from child workers in a streamed fashion

1010

11−module Streamly.Internal.Data.Stream.Channel.Worker

11+module Streamly.Internal.Data.Channel.Worker

1212 (

1313 Work (..)

1414 , estimateWorkers

@@ -34,7 +34,7 @@ import Streamly.Internal.Data.Time.Clock (Clock(Monotonic), getTime)

3434import Streamly.Internal.Data.Time.Units

3535 (AbsTime, NanoSecond64(..), diffAbsTime64, fromRelTime64)

3636

37−import Streamly.Internal.Data.Stream.Channel.Types

37+import Streamly.Internal.Data.Channel.Types

3838

3939-------------------------------------------------------------------------------

4040-- Yield control

src/Streamly/Internal/Data/Fold/Concurrent.hs+1−44

@@ -50,51 +50,8 @@

5050

5151module Streamly.Internal.Data.Fold.Concurrent

5252 (

53− -- * Configuration

54− Config

55− , maxBuffer

56− , bound

57− , inspect

58−

59− -- * Combinators

60− -- | Stream combinators using Async channel

61−

62− , parEval

53+ module Streamly.Internal.Data.Fold.Concurrent.Channel

6354 )

6455where

6556

66−-- import Streamly.Internal.Control.Concurrent (MonadAsync)

67−-- import Streamly.Internal.Data.Fold (Fold)

68−

6957import Streamly.Internal.Data.Fold.Concurrent.Channel

70−

71−-- $setup

72−--

73−-- Imports for example snippets in this module.

74−--

75−-- >>> :m

76−-- >>> :set -XFlexibleContexts

77−-- >>> import Control.Concurrent (threadDelay)

78−-- >>> import qualified Streamly.Data.Array as Array

79−-- >>> import qualified Streamly.Data.Fold as Fold

80−-- >>> import qualified Streamly.Data.Parser as Parser

81−-- >>> import qualified Streamly.Data.Stream as Stream

82−-- >>> import qualified Streamly.Internal.Data.Fold.Concurrent as Fold

83−-- >>> import Prelude hiding (concatMap, concat)

84−-- >>> :{

85−-- delay n = do

86−-- threadDelay (n * 1000000) -- sleep for n seconds

87−-- putStrLn (show n ++ " sec") -- print "n sec"

88−-- return n -- IO Int

89−-- :}

90−

91−{-

92−-- | Evaluate a stream asynchronously using a channel and serve the consumer

93−-- from the evaluation buffer.

94−--

95−-- >>> eval = Fold.parEval id

96−--

97−{-# INLINE eval #-}

98−eval :: MonadAsync m => Fold m a b -> Fold m a b

99−eval = parEval id

100−-}

src/Streamly/Internal/Data/Fold/Concurrent/Channel.hs+4−6

@@ -8,13 +8,11 @@

99module Streamly.Internal.Data.Fold.Concurrent.Channel

1010 (

11− -- * Channel

12− Channel

11+ module Streamly.Internal.Data.Fold.Concurrent.Channel.Type

1312

1413 -- * Configuration

15− , Config

1614 , maxBuffer

17− , bound

15+ , boundThreads

1816 , inspect

1917

2018 -- * Fold operations

@@ -28,11 +26,11 @@ import Control.Monad.IO.Class (MonadIO(liftIO))

2826import Data.IORef (writeIORef)

2927import Streamly.Internal.Control.Concurrent (MonadAsync)

3028import Streamly.Internal.Data.Fold (Fold(..), Step (..))

31−import Streamly.Internal.Data.Stream.Channel.Worker (sendWithDoorBell)

29+import Streamly.Internal.Data.Channel.Worker (sendWithDoorBell)

3230import Streamly.Internal.Data.Time.Clock (Clock(Monotonic), getTime)

3331

3432import Streamly.Internal.Data.Fold.Concurrent.Channel.Type

35−import Streamly.Internal.Data.Stream.Channel.Types

33+import Streamly.Internal.Data.Channel.Types

3634

3735-------------------------------------------------------------------------------

3836-- Evaluating a Fold

src/Streamly/Internal/Data/Fold/Concurrent/Channel/Type.hs+3−3

@@ -30,13 +30,13 @@ import Streamly.Internal.Control.Concurrent

3030 (MonadAsync, MonadRunInIO, askRunInIO)

3131import Streamly.Internal.Control.ForkLifted (doForkWith)

3232import Streamly.Internal.Data.Fold (Fold(..))

33−import Streamly.Internal.Data.Stream.Channel.Dispatcher (dumpSVarStats)

34−import Streamly.Internal.Data.Stream.Channel.Worker (sendWithDoorBell)

33+import Streamly.Internal.Data.Channel.Dispatcher (dumpSVarStats)

34+import Streamly.Internal.Data.Channel.Worker (sendWithDoorBell)

3535

3636import qualified Streamly.Internal.Data.Fold as Fold

3737import qualified Streamly.Internal.Data.Stream as D

3838

39−import Streamly.Internal.Data.Stream.Channel.Types

39+import Streamly.Internal.Data.Channel.Types

4040

4141data Channel m a b = Channel

4242 {

src/Streamly/Internal/Data/Fold/Prelude.hsnew file+19

@@ -0,0 +1,19 @@

1+-- |

2+-- Module : Streamly.Internal.Data.Fold.Prelude

4+-- License : BSD-3-Clause

5+-- Maintainer : streamly@composewell.com

6+-- Stability : experimental

7+-- Portability : GHC

8+--

9+module Streamly.Internal.Data.Fold.Prelude

10+ (

11+ module Streamly.Internal.Data.Fold.Async

12+ , module Streamly.Internal.Data.Fold.Concurrent

13+ , module Streamly.Internal.Data.Fold.SVar

14+ )

15+where

16+

17+import Streamly.Internal.Data.Fold.Async

18+import Streamly.Internal.Data.Fold.Concurrent

19+import Streamly.Internal.Data.Fold.SVar

src/Streamly/Internal/Data/Fold/SVar.hs+1

@@ -1,3 +1,4 @@

1+{-# OPTIONS_GHC -Wno-deprecations #-}

12-- |

23-- Module : Streamly.Internal.Data.Fold.SVar

src/Streamly/Internal/Data/IsMap/HashMap.hs+5

@@ -1,3 +1,8 @@

1+{-# LANGUAGE TypeFamilies #-}

2+-- Must come after TypeFamilies, otherwise it is re-enabled.

3+-- MonoLocalBinds enabled by TypeFamilies causes perf regressions in general.

4+{-# LANGUAGE NoMonoLocalBinds #-}

16{-# OPTIONS_GHC -Wno-orphans #-}

38-- |

src/Streamly/Internal/Data/SVar.hs+1

@@ -8,6 +8,7 @@

88--

99--

1010module Streamly.Internal.Data.SVar

11+ {-# DEPRECATED "SVar is replaced by Channel." #-}

1112 (

1213 module Streamly.Internal.Data.SVar.Type

1314 , module Streamly.Internal.Data.SVar.Worker

src/Streamly/Internal/Data/SmallArray.hs+1−1

@@ -106,7 +106,7 @@ foldr f z arr = runIdentity $ D.foldr f z $ toStreamD arr

106106-- of elements use an 'Array' from either "Streamly.Data.Array.Generic" or "Streamly.Data.Array.Foreign".

107107{-# INLINE_NORMAL writeN #-}

108108writeN :: MonadIO m => Int -> Fold m a (SmallArray a)

109−writeN len = FL.Fold step initial extract

109+writeN len = FL.Fold step initial extract extract

110110

111111 where

112112

core/src/Streamly/Internal/Data/Stream/Common.hs → src/Streamly/Internal/Data/Stream/Common.hsrenamed+3−3

@@ -30,10 +30,10 @@ where

3030

3131#include "inline.hs"

3232

33−import Streamly.Internal.Data.Fold.Type (Fold (..))

33+import Streamly.Data.Fold (Fold)

3434

35−import qualified Streamly.Internal.Data.StreamK.Type as K

36−import qualified Streamly.Internal.Data.Stream.Type as D

35+import qualified Streamly.Data.StreamK as K

36+import qualified Streamly.Internal.Data.Stream as D

3737

3838import Prelude hiding (Foldable(..), repeat)

3939

src/Streamly/Internal/Data/Stream/Concurrent.hs+50−39