git.iko.soy/project-m36
on master help sign in
fix CSV import/export encoding (now universally UTF-8)

all tests pass

commitdc3ee079a7295a266e140d8f081f9cd8b8309f4e authorAgentM <agentm@themactionfaction.com> dateThu 12 Apr 2018, 16:50 (9 years ago) parentbc2b0d1
5 files changed+20−8
compare with master
unifiedsplit
src/lib/ProjectM36/Atom.hs+1−1 src/lib/ProjectM36/Persist.hs+3−2 src/lib/ProjectM36/Relation/Show/CSV.hs+1−1 src/lib/ProjectM36/TransactionGraph/Persist.hs+14−3 test/Relation/Export/CSV.hs+1−1
src/lib/ProjectM36/Atom.hs+1−1
@@ -15,7 +15,7 @@ atomToText :: Atom -> T.Text
1515atomToText (IntegerAtom i) = (T.pack . show) i
1616atomToText (IntAtom i) = (T.pack . show) i
1717atomToText (DoubleAtom i) = (T.pack . show) i
18−atomToText (TextAtom i) = i --does this break quoting in CSV export?
18+atomToText (TextAtom i) = (T.pack . show) i --quotes necessary for ConstructedAtom subatoms
1919atomToText (DayAtom i) = (T.pack . show) i
2020atomToText (DateTimeAtom i) = (T.pack . show) i
2121atomToText (ByteStringAtom i) = (T.pack . show) i
src/lib/ProjectM36/Persist.hs+3−2
@@ -6,7 +6,6 @@ module ProjectM36.Persist (writeFileSync,
66 printFdCount,
77 DiskSync(..)) where
88-- on Windows, use FlushFileBuffers and MoveFileEx
9−import qualified Data.Text.IO as TIO
109import qualified Data.Text as T
1110
1211#if defined(linux_HOST_OS)
@@ -38,6 +37,8 @@ import Foreign.C
3837
3938import System.IO (withFile, IOMode(WriteMode), Handle)
4039import qualified Data.ByteString.Lazy as BS
40+import qualified Data.ByteString as BS'
41+import qualified Data.Text.Encoding as TE
4142
4243#if defined(mingw32_HOST_OS)
4344import ProjectM36.Win32Handle
@@ -52,7 +53,7 @@ writeFileSync :: DiskSync -> FilePath -> T.Text -> IO()
5253writeFileSync sync path strOut = withFile path WriteMode handler
5354 where
5455 handler handle = do
55− TIO.hPutStr handle strOut
56+ BS'.hPut handle (TE.encodeUtf8 strOut)
5657 syncHandle sync handle
5758
5859renameSync :: DiskSync -> FilePath -> FilePath -> IO ()
src/lib/ProjectM36/Relation/Show/CSV.hs+1−1
@@ -41,7 +41,7 @@ instance DefaultOrdered RecordRelationTuple where
4141newtype RecordAtom = RecordAtom {unAtom :: Atom}
4242
4343instance ToField RecordAtom where
44− toField (RecordAtom (TextAtom atomVal)) = TE.encodeUtf8 atomVal --squelch extraneous quotes for text type- the CSV library will add them if necessary
44+ toField (RecordAtom (TextAtom atomVal)) = TE.encodeUtf8 atomVal --without this, CSV text atoms are doubly quoted
4545 toField (RecordAtom atomVal) = (TE.encodeUtf8 . atomToText) atomVal
4646
4747
src/lib/ProjectM36/TransactionGraph/Persist.hs+14−3
@@ -21,7 +21,8 @@ import Data.Either (isRight)
2121import Data.Maybe (fromMaybe)
2222import qualified Data.List as L
2323import Control.Exception.Base
24−import qualified Data.Text.IO as TIO
24+import qualified Data.ByteString as BS
25+import qualified Data.Text.Encoding as TE
2526import Data.ByteString (ByteString)
2627import Data.Monoid
2728import qualified Crypto.Hash.SHA256 as SHA256
@@ -200,7 +201,7 @@ writeGraphTransactionIdFile sync destDirectory (TransactionGraph _ transSet) = w
200201
201202readGraphTransactionIdFileDigest :: FilePath -> IO LockFileHash
202203readGraphTransactionIdFileDigest dbdir = do
203− let graphTransactionIdData = TIO.readFile (transactionLogPath dbdir)
204+ let graphTransactionIdData = readUTF8FileOrError (transactionLogPath dbdir)
204205 SHA256.hash . encodeUtf8 <$> graphTransactionIdData
205206
206207readGraphTransactionIdFile :: FilePath -> IO (Either PersistenceError [(TransactionId, UTCTime, [TransactionId])])
@@ -210,5 +211,15 @@ readGraphTransactionIdFile dbdir = do
210211 (readUUID tid, readEpoch epochText, map readUUID parentIds)
211212 readUUID uuidText = fromMaybe (error "failed to read uuid") (U.fromText uuidText)
212213 readEpoch t = posixSecondsToUTCTime (realToFrac (either (error "failed to read epoch") fst (double t)))
213− Right . map grapher . T.lines <$> TIO.readFile (transactionLogPath dbdir)
214+ Right . map grapher . T.lines <$> readUTF8FileOrError (transactionLogPath dbdir)
214215
216+--rationale- reading essential database files must fail hard
217+readUTF8FileOrError :: FilePath -> IO (T.Text)
218+readUTF8FileOrError pathIn = do
219+ eFileBytes <- try (BS.readFile pathIn) :: IO (Either IOError BS.ByteString)
220+ case eFileBytes of
221+ Left err -> error (show err)
222+ Right fileBytes ->
223+ case TE.decodeUtf8' fileBytes of
224+ Left err -> error (show err)
225+ Right utf8Bytes -> pure utf8Bytes
test/Relation/Export/CSV.hs+1−1
@@ -85,7 +85,7 @@ testCSVExport = TestCase $ do
8585 Right rel ->
8686 case relationAsCSV rel of
8787 Left err -> assertFailure $ "export failed: " ++ show err
88− Right csvData ->
88+ Right csvData ->
8989 --BS.writeFile "/tmp/csv" csvData
9090 --putStrLn (TL.unpack (TE.decodeUtf8 csvData))
9191 case csvAsRelation attrs basicTypeConstructorMapping csvData of -- import csv data back to relation