git.iko.soy/typeable/cassava-streams
on trunk help sign in
21 files changed+587−71
since splitsnapshots
unifiedsplit
.envrc+10 .github/workflows/analysis.yml+22 .github/workflows/tests.yml+63 .gitignore+5−2 CHANGELOG → .gitmodules CHANGES.md+34 LICENSE+1−1 README.md+4 cassava-streams.cabal+55−43 data/HLint.hs+3 default.nix+11 nix/sources.json+38 nix/sources.nix+134 shell.nix+2 src/System/IO/Streams/Csv.hs+12 src/System/IO/Streams/Csv/Decode.hs+30−12 src/System/IO/Streams/Csv/Encode.hs+19−7 src/System/IO/Streams/Csv/Tutorial.hs+8−6 test/header.csv+1 test/make-huge-csv.sh+30 test/test.hs+105
.envrcnew file+10
@@ -0,0 +1,10 @@
1+# -*- sh -*-
2+
3+# Don't let cabal leak state to other projects:
4+export CABAL_DIR=$(pwd)/.cabal
5+
6+# Load in the shell.nix file:
7+use nix
8+
9+# Reload if these files change:
10+watch_file nix/sources.json
.github/workflows/analysis.ymlnew file+22
@@ -0,0 +1,22 @@
1+name: Analysis
2+on:
3+ pull_request:
4+jobs:
5+ check:
6+ name: pre-merge-checks
7+ runs-on: ubuntu-latest
8+ steps:
9+ - name: Clone Repository
10+ uses: actions/checkout@v2
11+
12+ - name: Setup Haskell
13+ id: setup-haskell-cabal
14+ uses: actions/setup-haskell@v1.1.3
15+ with:
16+ ghc-version: 8.8
17+
18+ - name: Cabal Check
19+ run: cabal check
20+
21+ - name: Lint
22+ run: curl -sSL https://raw.github.com/ndmitchell/hlint/master/misc/run.sh | sh -s .
.github/workflows/tests.ymlnew file+63
@@ -0,0 +1,63 @@
1+name: CI
2+on:
3+ pull_request:
4+jobs:
5+ cabal:
6+ name: ${{ matrix.os }} / ghc-${{ matrix.ghc }}
7+ runs-on: ${{ matrix.os }}
8+ strategy:
9+ fail-fast: false
10+ matrix:
11+ os: [ubuntu-latest, macOS-latest, windows-latest]
12+ cabal: ["3.2"]
13+ ghc: ["8.6.5", "8.8.4", "8.10.2"]
14+ # Newer versions of GHC are broken on Windows:
15+ # https://gitlab.haskell.org/ghc/ghc/-/issues/17926
16+ exclude:
17+ - os: windows-latest
18+ ghc: "8.8.4"
19+ - os: windows-latest
20+ ghc: "8.10.2"
21+ steps:
22+ - name: Clone Repository
23+ uses: actions/checkout@v2
24+
25+ - name: Setup Haskell
26+ id: setup-haskell-cabal
27+ uses: actions/setup-haskell@v1.1.3
28+ with:
29+ ghc-version: ${{ matrix.ghc }}
30+ cabal-version: ${{ matrix.cabal }}
31+
32+ - name: Configure and Freeze for Cache
33+ run: |
34+ cabal configure --enable-tests --enable-benchmarks --test-show-details=direct -ftutorial
35+ cabal freeze
36+
37+ - name: Cache ~/.cabal/store
38+ uses: actions/cache@v1
39+ with:
40+ path: ${{ steps.setup-haskell-cabal.outputs.cabal-store }}
41+ key: ${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('cabal.project.freeze') }}
42+
43+ - name: Build
44+ run: cabal build '--build-log=logs/$pkgid'
45+
46+ - name: Test
47+ run: cabal test
48+
49+ - name: Upload Build Logs on Failure
50+ uses: actions/upload-artifact@v2
51+ if: failure()
52+ with:
53+ name: logs-${{ matrix.os }}-ghc${{ matrix.ghc }}
54+ path: logs/
55+
56+ - name: Install
57+ run: cabal install --installdir=artifacts --install-method=copy
58+
59+ - name: Upload Artifacts
60+ uses: actions/upload-artifact@v2
61+ with:
62+ name: ${{ matrix.os }}-ghc${{ matrix.ghc }}
63+ path: artifacts/
.gitignore+5−2
@@ -1,3 +1,6 @@
1+/test/massive.csv
2+/.stack-work
3+/build/.stack-work
14/dist
2−/.cabal-sandbox
3−/cabal.sandbox.config
5+/result
6+/.cabal/
CHANGELOG → .gitmodulesrenamed

The content is the same.

CHANGES.mdnew file+34
@@ -0,0 +1,34 @@
1+# Change Log / Release Notes
2+
3+## 0.3 (August 11, 2016)
4+
5+ * Add some additional (convenience) exports from `Data.Csv`. (Thanks
6+ goes to Florian Hofmann.)
7+
8+ * Added `data/HLint.hs`
9+
10+ * Updated to LTS Haskell 6.7
11+
12+ * Minor updates:
13+
14+ - 0.3.0.1 (April 14, 2017): Updated dependencies
15+ - 0.3.0.2 (March 20, 2018): Updated dependencies
16+ - 0.3.0.3 (October 09, 2018): Updated dependencies
17+ - 0.3.0.4 (April 15, 2019): Updated dependencies
18+ - 0.3.0.5 (October 29, 2020): Update dependencies
19+
20+## 0.2.0.0 (February 8, 2016)
21+
22+ * Compile with GHC 7.10.3
23+
24+ * Update dependency versions
25+
26+ * Build with stack and LTS Haskell 5.2
27+
28+## 0.1.1.0 (November 11, 2014)
29+
30+ * `System/IO/Streams/Csv/Decode.hs` added more descriptive error messages.
31+
32+## 0.1.0.0 (April 30, 2014)
33+
34+ * cassava-streams initial release
LICENSE+1−1
@@ -1,4 +1,4 @@
1−Copyright (c) 2014 Peter Jones <pjones@devalot.com>
1+Copyright (c) 2014-2020 Peter Jones <pjones@devalot.com>
22
33All rights reserved.
44
README.md+4
@@ -1,5 +1,9 @@
11# cassava-streams
22
3+[![CI](https://github.com/pjones/cassava-streams/workflows/CI/badge.svg)](https://github.com/pjones/cassava-streams/actions)
4+[![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/pjones/cassava-streams?label=release)](https://github.com/pjones/cassava-streams/releases)
5+[![Hackage](https://img.shields.io/hackage/v/cassava-streams)](https://hackage.haskell.org/package/cassava-streams)
6+
37[io-streams][] interface for the [cassava][] CSV library.
48
59[io-streams]: http://hackage.haskell.org/package/io-streams
cassava-streams.cabal+55−43
@@ -1,18 +1,17 @@
1−--------------------------------------------------------------------------------
2−name: cassava-streams
3−version: 0.1.0.0
4−synopsis: io-streams interface for the cassava CSV library.
5−license: BSD3
6−license-file: LICENSE
7−author: Peter Jones <pjones@devalot.com>
8−maintainer: Peter Jones <pjones@devalot.com>
9−copyright: Copyright (c) 2014 Peter Jones
10−category: Data, Text, CSV, IO-Streams
11−build-type: Simple
12−cabal-version: >=1.10
13−tested-with: GHC==7.8.2
14−homepage: https://github.com/pjones/cassava-streams
15−bug-reports: https://github.com/pjones/cassava-streams/issues
1+cabal-version: 2.2
2+name: cassava-streams
3+version: 0.3.0.5
4+synopsis: io-streams interface for the cassava CSV library.
5+license: BSD-3-Clause
6+license-file: LICENSE
7+author: Peter Jones <pjones@devalot.com>
8+maintainer: Peter Jones <pjones@devalot.com>
9+copyright: Copyright (c) 2014-2020 Peter Jones
10+category: Data, Text, CSV, IO-Streams
11+build-type: Simple
12+homepage: https://github.com/pjones/cassava-streams
13+bug-reports: https://github.com/pjones/cassava-streams/issues
14+tested-with: GHC ==8.6.5 || ==8.8.4 || ==8.10.2
1615description:
1716 The cassava-streams library glues togeter the cassava CSV library
1817 and the io-streams streaming library.
@@ -22,58 +21,71 @@ description:
2221--------------------------------------------------------------------------------
2322extra-source-files:
2423 README.md
25− CHANGELOG
26− test/simple.csv
24+ CHANGES.md
25+ test/*.csv
2726
2827--------------------------------------------------------------------------------
2928source-repository head
3029 type: git
3130 location: https://github.com/pjones/cassava-streams.git
3231
33−--------------------------------------------------------------------------------
34−flag maintainer
35− description: Enable settings for the package maintainer.
36− default: False
37−
3832--------------------------------------------------------------------------------
3933flag tutorial
4034 description: Build the tutorial binary (useful for profiling).
41− default: False
35+ manual: True
36+ default: False
37+
38+--------------------------------------------------------------------------------
39+common dependencies
40+ build-depends:
41+ , base >=4.8 && <5.0
42+ , bytestring >=0.10 && <0.12
43+ , cassava >=0.4 && <0.6
44+ , io-streams >=1.1 && <1.6
45+ , vector >=0.10 && <0.14
4246
4347--------------------------------------------------------------------------------
4448library
49+ import: dependencies
4550 exposed-modules:
4651 System.IO.Streams.Csv
4752 System.IO.Streams.Csv.Tutorial
53+
4854 other-modules:
49− System.IO.Streams.Csv.Encode
5055 System.IO.Streams.Csv.Decode
56+ System.IO.Streams.Csv.Encode
5157
52− hs-source-dirs: src
58+ hs-source-dirs: src
5359 default-language: Haskell2010
54− ghc-options: -O2 -Wall -fwarn-incomplete-uni-patterns
55− ghc-prof-options: -prof -auto-all
56−
57− if flag(maintainer)
58− ghc-options: -Werror
59−
60− build-depends: base >= 4.6 && < 5.0
61− , bytestring >= 0.10 && < 1.0
62− , cassava >= 0.3 && < 0.4
63− , io-streams >= 1.1.0 && < 2.0
64− , vector >= 0.10 && < 1.0
60+ ghc-options: -Wall -fwarn-incomplete-uni-patterns
61+ build-depends:
6562
6663--------------------------------------------------------------------------------
6764executable tutorial
6865 default-language: Haskell2010
69− hs-source-dirs: bin
70− main-is: tutorial.hs
71− ghc-options: -O2 -Wall -Werror -rtsopts
72− ghc-prof-options: -fprof-auto-top -fprof-cafs
66+ hs-source-dirs: bin
67+ main-is: tutorial.hs
68+ ghc-options: -Wall -fwarn-incomplete-uni-patterns -rtsopts
7369
7470 if !flag(tutorial)
7571 buildable: False
72+
7673 else
77− build-depends: base
78− , cassava-streams
79− , io-streams
74+ build-depends:
75+ , base
76+ , cassava-streams
77+ , io-streams
78+
79+--------------------------------------------------------------------------------
80+test-suite test
81+ import: dependencies
82+ type: exitcode-stdio-1.0
83+ default-language: Haskell2010
84+ hs-source-dirs: test
85+ main-is: test.hs
86+ ghc-options: -Wall -fwarn-incomplete-uni-patterns
87+ build-depends:
88+ , cassava-streams
89+ , QuickCheck >=2.7 && <3.0
90+ , tasty >=0.8 && <1.5
91+ , tasty-quickcheck >=0.8 && <1.3
data/HLint.hsnew file+3
@@ -0,0 +1,3 @@
1+module HLint.HLint where
2+import "hint" HLint.Builtin.All
3+ignore "Use import/export shortcut"
default.nixnew file+11
@@ -0,0 +1,11 @@
1+{ sources ? import ./nix/sources.nix
2+, pkgs ? import sources.nixpkgs { }
3+, nix-hs ? import sources.nix-hs { inherit pkgs; }
4+, ghc ? "default"
5+}:
6+
7+nix-hs {
8+ cabal = ./cassava-streams.cabal;
9+ flags = [ "tutorial" ];
10+ compiler = ghc;
11+}
nix/sources.jsonnew file+38
@@ -0,0 +1,38 @@
1+{
2+ "niv": {
3+ "branch": "master",
4+ "description": "Easy dependency management for Nix projects",
5+ "homepage": "https://github.com/nmattia/niv",
6+ "owner": "nmattia",
7+ "repo": "niv",
8+ "rev": "f73bf8d584148677b01859677a63191c31911eae",
9+ "sha256": "0jlmrx633jvqrqlyhlzpvdrnim128gc81q5psz2lpp2af8p8q9qs",
10+ "type": "tarball",
11+ "url": "https://github.com/nmattia/niv/archive/f73bf8d584148677b01859677a63191c31911eae.tar.gz",
12+ "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
13+ },
14+ "nix-hs": {
15+ "branch": "release-20.09",
16+ "description": "Haskell + nixpkgs = nix-hs",
17+ "homepage": "",
18+ "owner": "pjones",
19+ "repo": "nix-hs",
20+ "rev": "67163bb02a8791d79136ba6914b6d6dad594533b",
21+ "sha256": "0f7sjh45wxbf2m76l37f4l4v4dkq32dm599p85acx53324nr9y8z",
22+ "type": "tarball",
23+ "url": "https://github.com/pjones/nix-hs/archive/67163bb02a8791d79136ba6914b6d6dad594533b.tar.gz",
24+ "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
25+ },
26+ "nixpkgs": {
27+ "branch": "nixos-20.09",
28+ "description": "Nix Packages collection",
29+ "homepage": "",
30+ "owner": "NixOS",
31+ "repo": "nixpkgs",
32+ "rev": "edb26126d98bc696f4f3e206583faa65d3d6e818",
33+ "sha256": "1cl4ka4kk7kh3bl78g06dhiidazf65q8miyzaxi9930d6gwyzkci",
34+ "type": "tarball",
35+ "url": "https://github.com/NixOS/nixpkgs/archive/edb26126d98bc696f4f3e206583faa65d3d6e818.tar.gz",
36+ "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
37+ }
38+}
nix/sources.nixnew file+134
@@ -0,0 +1,134 @@
1+# This file has been generated by Niv.
2+
3+let
4+
5+ #
6+ # The fetchers. fetch_<type> fetches specs of type <type>.
7+ #
8+
9+ fetch_file = pkgs: spec:
10+ if spec.builtin or true then
11+ builtins_fetchurl { inherit (spec) url sha256; }
12+ else
13+ pkgs.fetchurl { inherit (spec) url sha256; };
14+
15+ fetch_tarball = pkgs: spec:
16+ if spec.builtin or true then
17+ builtins_fetchTarball { inherit (spec) url sha256; }
18+ else
19+ pkgs.fetchzip { inherit (spec) url sha256; };
20+
21+ fetch_git = spec:
22+ builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
23+
24+ fetch_builtin-tarball = spec:
25+ builtins.trace
26+ ''
27+ WARNING:
28+ The niv type "builtin-tarball" will soon be deprecated. You should
29+ instead use `builtin = true`.
30+
31+ $ niv modify <package> -a type=tarball -a builtin=true
32+ ''
33+ builtins_fetchTarball { inherit (spec) url sha256; };
34+
35+ fetch_builtin-url = spec:
36+ builtins.trace
37+ ''
38+ WARNING:
39+ The niv type "builtin-url" will soon be deprecated. You should
40+ instead use `builtin = true`.
41+
42+ $ niv modify <package> -a type=file -a builtin=true
43+ ''
44+ (builtins_fetchurl { inherit (spec) url sha256; });
45+
46+ #
47+ # Various helpers
48+ #
49+
50+ # The set of packages used when specs are fetched using non-builtins.
51+ mkPkgs = sources:
52+ let
53+ sourcesNixpkgs =
54+ import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {};
55+ hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
56+ hasThisAsNixpkgsPath = <nixpkgs> == ./.;
57+ in
58+ if builtins.hasAttr "nixpkgs" sources
59+ then sourcesNixpkgs
60+ else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
61+ import <nixpkgs> {}
62+ else
63+ abort
64+ ''
65+ Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
66+ add a package called "nixpkgs" to your sources.json.
67+ '';
68+
69+ # The actual fetching function.
70+ fetch = pkgs: name: spec:
71+
72+ if ! builtins.hasAttr "type" spec then
73+ abort "ERROR: niv spec ${name} does not have a 'type' attribute"
74+ else if spec.type == "file" then fetch_file pkgs spec
75+ else if spec.type == "tarball" then fetch_tarball pkgs spec
76+ else if spec.type == "git" then fetch_git spec
77+ else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec
78+ else if spec.type == "builtin-url" then fetch_builtin-url spec
79+ else
80+ abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
81+
82+ # Ports of functions for older nix versions
83+
84+ # a Nix version of mapAttrs if the built-in doesn't exist
85+ mapAttrs = builtins.mapAttrs or (
86+ f: set: with builtins;
87+ listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
88+ );
89+
90+ # fetchTarball version that is compatible between all the versions of Nix
91+ builtins_fetchTarball = { url, sha256 }@attrs:
92+ let
93+ inherit (builtins) lessThan nixVersion fetchTarball;
94+ in
95+ if lessThan nixVersion "1.12" then
96+ fetchTarball { inherit url; }
97+ else
98+ fetchTarball attrs;
99+
100+ # fetchurl version that is compatible between all the versions of Nix
101+ builtins_fetchurl = { url, sha256 }@attrs:
102+ let
103+ inherit (builtins) lessThan nixVersion fetchurl;
104+ in
105+ if lessThan nixVersion "1.12" then
106+ fetchurl { inherit url; }
107+ else
108+ fetchurl attrs;
109+
110+ # Create the final "sources" from the config
111+ mkSources = config:
112+ mapAttrs (
113+ name: spec:
114+ if builtins.hasAttr "outPath" spec
115+ then abort
116+ "The values in sources.json should not have an 'outPath' attribute"
117+ else
118+ spec // { outPath = fetch config.pkgs name spec; }
119+ ) config.sources;
120+
121+ # The "config" used by the fetchers
122+ mkConfig =
123+ { sourcesFile ? ./sources.json
124+ , sources ? builtins.fromJSON (builtins.readFile sourcesFile)
125+ , pkgs ? mkPkgs sources
126+ }: rec {
127+ # The sources, i.e. the attribute set of spec name to spec
128+ inherit sources;
129+
130+ # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
131+ inherit pkgs;
132+ };
133+in
134+mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }
shell.nixnew file+2
@@ -0,0 +1,2 @@
1+# Load an interactive environment:
2+(import ./. {}).interactive
src/System/IO/Streams/Csv.hs+12
@@ -41,8 +41,20 @@ module System.IO.Streams.Csv
4141 -- ByteString@ stream into one that encodes records into CSV
4242 -- format before sending them downstream.
4343 , module System.IO.Streams.Csv.Encode
44+
45+ -- * Convenience Exports
46+ -- | Export data types from Data.Csv
47+ , module Data.Csv
4448 ) where
4549
4650--------------------------------------------------------------------------------
4751import System.IO.Streams.Csv.Decode
4852import System.IO.Streams.Csv.Encode
53+
54+import Data.Csv ( HasHeader(..)
55+ , defaultEncodeOptions
56+ , defaultDecodeOptions
57+ , DecodeOptions(..)
58+ , EncodeOptions(..)
59+ , Quoting(..)
60+ )
src/System/IO/Streams/Csv/Decode.hs+30−12
@@ -1,3 +1,6 @@
1+{-# LANGUAGE DeriveDataTypeable #-}
2+{-# LANGUAGE ScopedTypeVariables #-}
3+
14{-
25
36This file is part of the Haskell package cassava-streams. It is
@@ -12,7 +15,8 @@ file.
1215
1316--------------------------------------------------------------------------------
1417module System.IO.Streams.Csv.Decode
15− ( decodeStream
18+ ( StreamDecodingError (..)
19+ , decodeStream
1620 , decodeStreamWith
1721 , decodeStreamByName
1822 , decodeStreamByNameWith
@@ -20,14 +24,24 @@ module System.IO.Streams.Csv.Decode
2024 ) where
2125
2226--------------------------------------------------------------------------------
27+import Control.Exception
2328import Data.ByteString (ByteString)
2429import qualified Data.ByteString as BS
2530import Data.Csv hiding (Parser, decodeWith, decodeByNameWith)
2631import Data.Csv.Incremental
2732import Data.IORef
33+import Data.Typeable
2834import System.IO.Streams (InputStream, makeInputStream)
2935import qualified System.IO.Streams as Streams
3036
37+--------------------------------------------------------------------------------
38+-- | Exception thrown when stream decoding cannot continue due to an
39+-- error.
40+newtype StreamDecodingError = StreamDecodingError String
41+ deriving (Typeable, Show)
42+
43+instance Exception StreamDecodingError
44+
3145--------------------------------------------------------------------------------
3246-- | Create an @InputStream@ which decodes CSV records from the given
3347-- upstream data source.
@@ -103,12 +117,12 @@ onlyValidRecords input = makeInputStream $ do
103117
104118 case upstream of
105119 Nothing -> return Nothing
106− Just (Left err) -> bomb err -- FIXME: replace with throwIO
120+ Just (Left err) -> bomb ("invalid CSV row: " ++ err)
107121 Just (Right x) -> return (Just x)
108122
109123--------------------------------------------------------------------------------
110124-- | Internal function which feeds data to the CSV parser.
111−dispatch :: IORef [Either String a]
125+dispatch :: forall a. IORef [Either String a]
112126 -- ^ List of queued CSV records.
113127 -> IORef (Maybe (Parser a))
114128 -- ^ Current CSV parser state.
@@ -124,9 +138,10 @@ dispatch queueRef parserRef input = do
124138 parser <- readIORef parserRef
125139 case parser of
126140 Nothing -> return Nothing
127− Just (Fail _ e) -> bomb e
128− Just (Many xs f) -> more f >> feed xs
129− Just (Done xs ) -> writeIORef parserRef Nothing >> feed xs
141+ Just (Fail _ e) -> bomb ("input data malformed: " ++ e)
142+ Just (Many [] f) -> more f
143+ Just (Many xs f) -> writeIORef parserRef (Just $ Many [] f) >> feed xs
144+ Just (Done xs ) -> writeIORef parserRef Nothing >> feed xs
130145
131146 (x:xs) -> do
132147 writeIORef queueRef xs
@@ -135,14 +150,17 @@ dispatch queueRef parserRef input = do
135150 where
136151 -- Send more data to the CSV parser. If there is no more data
137152 -- from upstream then send an empty @ByteString@.
138− more f = Streams.read input >>=
139− writeIORef parserRef . Just . maybe (f BS.empty) f
153+ more :: (ByteString -> Parser a) -> IO (Maybe (Either String a))
154+ more f = do bs <- Streams.read input
155+ writeIORef parserRef (Just $ maybe (f BS.empty) f bs)
156+ dispatch queueRef parserRef input
140157
141158 -- Feed records downstream.
142− feed xs = writeIORef queueRef xs >>
143− dispatch queueRef parserRef input
159+ feed :: [Either String a] -> IO (Maybe (Either String a))
160+ feed xs = do writeIORef queueRef xs
161+ dispatch queueRef parserRef input
144162
145163--------------------------------------------------------------------------------
146−-- | FIXME: use a proper exception.
164+-- | Throw an exception.
147165bomb :: String -> IO a
148−bomb = fail
166+bomb = throwIO . StreamDecodingError
src/System/IO/Streams/Csv/Encode.hs+19−7
@@ -19,9 +19,11 @@ module System.IO.Streams.Csv.Encode
1919 ) where
2020
2121--------------------------------------------------------------------------------
22+import Control.Monad (when)
2223import Data.ByteString (ByteString)
2324import qualified Data.ByteString.Lazy as BL
2425import Data.Csv
26+import Data.IORef
2527import System.IO.Streams (OutputStream, makeOutputStream)
2628import qualified System.IO.Streams as Streams
2729
@@ -44,7 +46,9 @@ encodeStreamWith :: ToRecord a
4446 => EncodeOptions -- ^ Encoding options.
4547 -> OutputStream ByteString -- ^ Downstream.
4648 -> IO (OutputStream a) -- ^ New @OutputStream@.
47−encodeStreamWith ops = makeOutputStream . dispatch (encodeWith ops)
49+encodeStreamWith opts output = do
50+ ref <- newIORef opts
51+ makeOutputStream (dispatch encodeWith ref output)
4852
4953--------------------------------------------------------------------------------
5054-- | Create a new @OutputStream@ which can be fed @ToNamedRecord@
@@ -69,11 +73,19 @@ encodeStreamByNameWith :: ToNamedRecord a
6973 -> Header -- ^ CSV Header.
7074 -> OutputStream ByteString -- ^ Downstream.
7175 -> IO (OutputStream a) -- ^ New @OutputStream@.
72−encodeStreamByNameWith ops hdr =
73− makeOutputStream . dispatch (encodeByNameWith ops hdr)
76+encodeStreamByNameWith opts hdr output = do
77+ ref <- newIORef opts
78+ makeOutputStream $ dispatch (`encodeByNameWith` hdr) ref output
7479
7580--------------------------------------------------------------------------------
76−-- | Do the actual encoding.
77−dispatch :: ([a] -> BL.ByteString) -> OutputStream ByteString -> Maybe a -> IO ()
78−dispatch _ output Nothing = Streams.write Nothing output
79−dispatch enc output (Just x) = Streams.writeLazyByteString (enc [x]) output
81+-- | Encode records, ensuring that the header is written no more than once.
82+dispatch :: (EncodeOptions -> [a] -> BL.ByteString) -- ^ Encoding function.
83+ -> IORef EncodeOptions -- ^ Encoding options.
84+ -> OutputStream ByteString -- ^ Downstream.
85+ -> Maybe a -- ^ Record to write.
86+ -> IO ()
87+dispatch _ _ output Nothing = Streams.write Nothing output
88+dispatch enc ref output (Just x) = do
89+ opts <- readIORef ref
90+ when (encIncludeHeader opts) $ writeIORef ref (opts {encIncludeHeader = False})
91+ Streams.writeLazyByteString (enc opts [x]) output
src/System/IO/Streams/Csv/Tutorial.hs+8−6
@@ -30,7 +30,6 @@ module System.IO.Streams.Csv.Tutorial
3030 ) where
3131
3232--------------------------------------------------------------------------------
33−import Control.Applicative
3433import Control.Monad
3534import Data.Csv
3635import qualified Data.Vector as V
@@ -41,18 +40,21 @@ import System.IO.Streams.Csv
4140--------------------------------------------------------------------------------
4241-- | A to-do item.
4342data Item = Item
44− { title :: String
45− , state :: TState
43+ { title :: String -- ^ Title.
44+ , state :: TState -- ^ State.
45+ , time :: Maybe Double -- ^ Seconds taken to complete.
4646 } deriving (Show, Eq)
4747
4848instance FromNamedRecord Item where
4949 parseNamedRecord m = Item <$> m .: "Title"
5050 <*> m .: "State"
51+ <*> m .: "Time"
5152
5253instance ToNamedRecord Item where
53− toNamedRecord (Item t s) =
54+ toNamedRecord (Item t s tm) =
5455 namedRecord [ "Title" .= t
5556 , "State" .= s
57+ , "Time" .= tm
5658 ]
5759
5860--------------------------------------------------------------------------------
@@ -105,7 +107,7 @@ onlyTodo inH outH = do
105107
106108 -- A stream to write items into. They will be converted to CSV.
107109 output <- Streams.handleToOutputStream outH >>=
108− encodeStreamByName (V.fromList ["State", "Title"])
110+ encodeStreamByName (V.fromList ["State", "Time", "Title"])
109111
110112 -- Connect the input and output streams.
111113 Streams.connect input output
@@ -132,7 +134,7 @@ markDone titleOfItem inH outH = do
132134
133135 -- A stream to write items into. They will be converted to CSV.
134136 output <- Streams.handleToOutputStream outH >>=
135− encodeStreamByName (V.fromList ["State", "Title"])
137+ encodeStreamByName (V.fromList ["State", "Time", "Title"])
136138
137139 -- Connect the input and output streams.
138140 Streams.connect input output
test/header.csvnew file+1
@@ -0,0 +1 @@
1+State,Time,Title
test/make-huge-csv.shnew file+30
@@ -0,0 +1,30 @@
1+#!/bin/sh
2+
3+################################################################################
4+PROF_ROW_SIZE=512
5+PROF_ROW_COUNT=10000 # Actual rows will be this number * 100
6+
7+################################################################################
8+OUT_FILE=$1
9+CSV_HEADER=$2
10+
11+################################################################################
12+cp $CSV_HEADER $OUT_FILE
13+
14+################################################################################
15+for i in `seq $PROF_ROW_COUNT`; do
16+ title=`openssl rand -base64 $PROF_ROW_SIZE | paste -sd/`
17+ time=`echo|awk 'srand() {print rand()}'`
18+
19+ if [ `expr $i % 2` -eq 0 ]; then
20+ row="TODO,,$title"
21+ else
22+ row="TODO,$time,$title"
23+ fi
24+
25+ for j in `seq 100`; do echo "$row" >> $OUT_FILE; done
26+ printf .
27+done
28+
29+################################################################################
30+echo # Print blank line to terminate all those dots.
test/test.hsnew file+105
@@ -0,0 +1,105 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+
3+{-
4+
5+This file is part of the Haskell package cassava-streams. It is
6+subject to the license terms in the LICENSE file found in the
7+top-level directory of this distribution and at
8+git://pmade.com/cassava-streams/LICENSE. No part of cassava-streams
9+package, including this file, may be copied, modified, propagated, or
10+distributed except according to the terms contained in the LICENSE
11+file.
12+
13+-}
14+
15+--------------------------------------------------------------------------------
16+module Main (main) where
17+
18+--------------------------------------------------------------------------------
19+import Control.Monad
20+import Data.ByteString (ByteString)
21+import qualified Data.ByteString as BS
22+import Data.Csv hiding (Record, NamedRecord, record, header)
23+import qualified Data.Vector as V
24+import System.IO.Streams (InputStream, OutputStream)
25+import qualified System.IO.Streams as Streams
26+import qualified System.IO.Streams.Csv as CSV
27+import Test.QuickCheck.Monadic (monadicIO, run, assert)
28+import Test.Tasty
29+import Test.Tasty.QuickCheck as QC
30+
31+--------------------------------------------------------------------------------
32+-- | Fake record to encode and decode. This works well because
33+-- Cassava and QuickCheck already have the necessary instances for
34+-- triples.
35+type Record = (Int, String, String)
36+
37+--------------------------------------------------------------------------------
38+-- | But, Cassava doesn't have ToNamedRecord, FromNamedRecord
39+-- instances for triples so we have to work around there here.
40+newtype NamedRecord = NamedRecord {record :: Record}
41+
42+instance ToNamedRecord NamedRecord where
43+ toNamedRecord (NamedRecord (a, b, c)) =
44+ namedRecord ["a" .= a, "b" .= b, "c" .= c]
45+
46+instance FromNamedRecord NamedRecord where
47+ parseNamedRecord m = do
48+ a <- m .: "a"
49+ b <- m .: "b"
50+ c <- m .: "c"
51+ return $ NamedRecord (a, b, c)
52+
53+
54+--------------------------------------------------------------------------------
55+header :: Header
56+header = V.fromList ["a", "b", "c"]
57+
58+--------------------------------------------------------------------------------
59+-- | Given a list of records generated by QuickCheck, encode those
60+-- records into a CSV ByteString then decode them back into records.
61+roundTrip :: (InputStream ByteString -> IO (InputStream a)) -- ^ Decoder.
62+ -> (OutputStream ByteString -> IO (OutputStream a)) -- ^ Encoder.
63+ -> [a] -- ^ Records.
64+ -> IO [a]
65+roundTrip is os recs = do
66+ -- Encode records to a ByteString.
67+ sourceList <- Streams.fromList recs
68+ (collector, encoded) <- Streams.listOutputStream
69+ encoder <- os collector
70+ Streams.connect sourceList encoder
71+
72+ -- Decode from ByteString.
73+ decoder <- (encoded >>= Streams.fromByteString . BS.concat) >>= is
74+ (decodeStream, decoded) <- Streams.listOutputStream
75+ Streams.connect decoder decodeStream
76+ decoded
77+
78+--------------------------------------------------------------------------------
79+prop_namedRoundTrip :: [Record] -> Property
80+prop_namedRoundTrip recsIn = not (null recsIn) ==> monadicIO $ do
81+ recsOut <- run $ roundTrip is os (map NamedRecord recsIn)
82+ assert $ recsIn == map record recsOut
83+ where
84+ is = CSV.decodeStreamByName >=> CSV.onlyValidRecords
85+ os = CSV.encodeStreamByName header
86+
87+--------------------------------------------------------------------------------
88+prop_indexedRoundTrip :: [Record] -> Property
89+prop_indexedRoundTrip recsIn = not (null recsIn) ==> monadicIO $ do
90+ recsOut <- run $ roundTrip is os recsIn
91+ assert $ recsIn == recsOut
92+ where
93+ is = CSV.decodeStream NoHeader >=> CSV.onlyValidRecords
94+ os = CSV.encodeStream
95+
96+--------------------------------------------------------------------------------
97+tests :: TestTree
98+tests = testGroup "Tests"
99+ [ QC.testProperty "namedRoundTrip" prop_namedRoundTrip
100+ , QC.testProperty "indexedRoundTrip" prop_indexedRoundTrip
101+ ]
102+
103+--------------------------------------------------------------------------------
104+main :: IO ()
105+main = defaultMain tests