ba7f69c · typeable/cassava-streams

git.iko.soy/typeable/cassava-streams

Add better error messages, small workflow changes

commitba7f69c731fd9fd0febaf0e95725e790f0d2a094 authorPeter Jones <pjones@pmade.com> dateTue 11 Nov 2014, 12:12 (12 years ago) parent2053883

7 files changed+62−29

compare with trunk

unifiedsplit

CHANGELOG+5 GNUmakefile+2−12 cassava-streams.cabal+2−2 src/System/IO/Streams/Csv/Decode.hs+14−9 src/System/IO/Streams/Csv/Tutorial.hs+8−5 test/header.csv+1−1 test/make-huge-csv.sh+30

CHANGELOG+5

@@ -1,3 +1,8 @@

1+2014-11-11 Peter Jones <pjones@devalot.com>

3+ * System/IO/Streams/Csv/Decode.hs added more descriptive error messages.

4+ * cassava-streams bumped version number (0.1.1.0).

162014-04-30 Peter Jones <pjones@devalot.com>

38 * cassava-streams initial release (version 0.1.0.0).

GNUmakefile+2−12

@@ -8,10 +8,6 @@ CSV_HEADER = test/header.csv

88CSV_MEGA = test/massive.csv

99TUTORIAL = dist/build/tutorial/tutorial todo

1010PROF_OPTS = +RTS -sstderr -hc -pa -xc

11−PROF_ROW_SIZE = 512

12−PROF_ROW_COUNT = 10000 # Actual rows will be this number * 100

13−RAND_ROW = openssl rand -base64 $(PROF_ROW_SIZE) | paste -sd/

14−

1511

1612################################################################################

1713clean::

@@ -20,18 +16,12 @@ clean::

2016

2117################################################################################

2218profile:: $(CSV_MEGA)

23− $(TUTORIAL) $(PROF_OPTS) < $(CSV_MEGA) > /dev/null

19+ $(TUTORIAL) $(PROF_OPTS) < $(CSV_MEGA) > profile.stdout

2420 hp2ps -M -b -c tutorial.hp

2521 ps2pdf tutorial.ps

2622

2723################################################################################

2824# Generate a very large CSV file for profiling.

2925$(CSV_MEGA): $(CSV_HEADER)

30− cp $(CSV_HEADER) $@

31− for i in `seq $(PROF_ROW_COUNT)`; do \

32− row=`printf "TODO,"; $(RAND_ROW)`; \

33− for j in `seq 100`; do echo "$$row" >> $@; done; \

34− printf .; \

35− done

36− @ echo # print a newline

26+ sh test/make-huge-csv.sh $@ $(CSV_HEADER)

3727 du -h $@

cassava-streams.cabal+2−2

@@ -1,6 +1,6 @@

11--------------------------------------------------------------------------------

22name: cassava-streams

3−version: 0.1.0.0

3+version: 0.1.1.0

44synopsis: io-streams interface for the cassava CSV library.

55license: BSD3

66license-file: LICENSE

1010category: Data, Text, CSV, IO-Streams

1111build-type: Simple

1212cabal-version: >=1.10

13−tested-with: GHC==7.8.2

13+tested-with: GHC==7.8.2, GHC==7.8.3

1414homepage: https://github.com/pjones/cassava-streams

1515bug-reports: https://github.com/pjones/cassava-streams/issues

1616description:

src/System/IO/Streams/Csv/Decode.hs+14−9

@@ -1,4 +1,5 @@

11{-# LANGUAGE DeriveDataTypeable #-}

2+{-# LANGUAGE ScopedTypeVariables #-}

34{-

@@ -116,12 +117,12 @@ onlyValidRecords input = makeInputStream $ do

116117

117118 case upstream of

118119 Nothing -> return Nothing

119− Just (Left err) -> bomb err

120+ Just (Left err) -> bomb ("invalid CSV row: " ++ err)

120121 Just (Right x) -> return (Just x)

121122

122123--------------------------------------------------------------------------------

123124-- | Internal function which feeds data to the CSV parser.

124−dispatch :: IORef [Either String a]

125+dispatch :: forall a. IORef [Either String a]

125126 -- ^ List of queued CSV records.

126127 -> IORef (Maybe (Parser a))

127128 -- ^ Current CSV parser state.

@@ -137,9 +138,10 @@ dispatch queueRef parserRef input = do

137138 parser <- readIORef parserRef

138139 case parser of

139140 Nothing -> return Nothing

140− Just (Fail _ e) -> bomb e

141− Just (Many xs f) -> more f >> feed xs

142− Just (Done xs ) -> writeIORef parserRef Nothing >> feed xs

141+ Just (Fail _ e) -> bomb ("input data malformed: " ++ e)

142+ Just (Many [] f) -> more f

143+ Just (Many xs f) -> writeIORef parserRef (Just $ Many [] f) >> feed xs

144+ Just (Done xs ) -> writeIORef parserRef Nothing >> feed xs

143145

144146 (x:xs) -> do

145147 writeIORef queueRef xs

@@ -148,12 +150,15 @@ dispatch queueRef parserRef input = do

148150 where

149151 -- Send more data to the CSV parser. If there is no more data

150152 -- from upstream then send an empty @ByteString@.

151− more f = Streams.read input >>=

152− writeIORef parserRef . Just . maybe (f BS.empty) f

153+ more :: (ByteString -> Parser a) -> IO (Maybe (Either String a))

154+ more f = do bs <- Streams.read input

155+ writeIORef parserRef (Just $ maybe (f BS.empty) f bs)

156+ dispatch queueRef parserRef input

153157

154158 -- Feed records downstream.

155− feed xs = writeIORef queueRef xs >>

156− dispatch queueRef parserRef input

159+ feed :: [Either String a] -> IO (Maybe (Either String a))

160+ feed xs = do writeIORef queueRef xs

161+ dispatch queueRef parserRef input

157162

158163--------------------------------------------------------------------------------

159164-- | Throw an exception.

src/System/IO/Streams/Csv/Tutorial.hs+8−5

@@ -41,18 +41,21 @@ import System.IO.Streams.Csv

4141--------------------------------------------------------------------------------

4242-- | A to-do item.

4343data Item = Item

44− { title :: String

45− , state :: TState

44+ { title :: String -- ^ Title.

45+ , state :: TState -- ^ State.

46+ , time :: Maybe Double -- ^ Seconds taken to complete.

4647 } deriving (Show, Eq)

4748

4849instance FromNamedRecord Item where

4950 parseNamedRecord m = Item <$> m .: "Title"

5051 <*> m .: "State"

52+ <*> m .: "Time"

5153

5254instance ToNamedRecord Item where

53− toNamedRecord (Item t s) =

55+ toNamedRecord (Item t s tm) =

5456 namedRecord [ "Title" .= t

5557 , "State" .= s

58+ , "Time" .= tm

5659 ]

5760

5861--------------------------------------------------------------------------------

@@ -105,7 +108,7 @@ onlyTodo inH outH = do

105108

106109 -- A stream to write items into. They will be converted to CSV.

107110 output <- Streams.handleToOutputStream outH >>=

108− encodeStreamByName (V.fromList ["State", "Title"])

111+ encodeStreamByName (V.fromList ["State", "Time", "Title"])

109112

110113 -- Connect the input and output streams.

111114 Streams.connect input output

@@ -132,7 +135,7 @@ markDone titleOfItem inH outH = do

132135

133136 -- A stream to write items into. They will be converted to CSV.

134137 output <- Streams.handleToOutputStream outH >>=

135− encodeStreamByName (V.fromList ["State", "Title"])

138+ encodeStreamByName (V.fromList ["State", "Time", "Title"])

136139

137140 -- Connect the input and output streams.

138141 Streams.connect input output

test/header.csv+1−1

@@ -1 +1 @@

1−State,Title

1+State,Time,Title

test/make-huge-csv.shnew file+30

@@ -0,0 +1,30 @@

1+#!/bin/sh

3+################################################################################

4+PROF_ROW_SIZE=512

5+PROF_ROW_COUNT=10000 # Actual rows will be this number * 100

7+################################################################################

8+OUT_FILE=$1

9+CSV_HEADER=$2

10+

11+################################################################################

12+cp $CSV_HEADER $OUT_FILE

13+

14+################################################################################

15+for i in `seq $PROF_ROW_COUNT`; do

16+ title=`openssl rand -base64 $PROF_ROW_SIZE | paste -sd/`

17+ time=`echo|awk 'srand() {print rand()}'`

18+

19+ if [ `expr $i % 2` -eq 0 ]; then

20+ row="TODO,,$title"

21+ else

22+ row="TODO,$time,$title"

23+ fi

24+

25+ for j in `seq 100`; do echo "$row" >> $OUT_FILE; done

26+ printf .

27+done

28+

29+################################################################################

30+echo # Print blank line to terminate all those dots.