git.iko.soy/project-m36
on master help sign in
update bigrel with three tuple data structure options
26c10d3AgentM
ProjectM36/bigrel.hs+46−5
compare with master
unifiedsplit
@@ -1,4 +1,4 @@
1−
1+{-# LANGUAGE FlexibleInstances #-}
2import ProjectM36.Base
2import ProjectM36.Base
3import ProjectM36.Relation
3import ProjectM36.Relation
4import ProjectM36.Tuple
4import ProjectM36.Tuple
@@ -7,8 +7,49 @@ import ProjectM36.Relation.Show.CSV
7import qualified Data.Map as M
7import qualified Data.Map as M
8import qualified Data.HashSet as HS
8import qualified Data.HashSet as HS
9import qualified Data.ByteString.Lazy.Char8 as BS
9import qualified Data.ByteString.Lazy.Char8 as BS
10+import qualified Data.IntMap as IM
11+import qualified Data.Hashable as Hash
12+import qualified Data.Vector as V
13+import Control.DeepSeq
10
14
11−main = do
15+dumpcsv :: Relation -> IO ()
16+dumpcsv = BS.putStrLn . relationAsCSV
17+
18+main = vectorMatrixRun
19+ --intmapMatrixRun
20+
21+-- takes 30 minutes to run and 1.1 GB
22+matrixRun = do
12− case matrixRelation 10 100000 of
23+ case matrixRelation 100 100000 of
13− Right rel -> BS.putStrLn $ relationAsCSV rel
24+ Left err -> putStrLn (show err)
14− Left err -> putStrLn (show err)
25+ Right rel -> case restrict (\(RelationTuple tupMap) -> tupMap M.! "0" == IntAtom 5) rel of
26+ Left err -> putStrLn (show err)
27+ Right rel -> dumpcsv rel
28+
29+intmapMatrixRun = do
30+ let matrix = intmapMatrixRelation 100 100000
31+ putStrLn (show matrix)
32+
33+--compare IntMap speed and size
34+--this is about 3 times faster (9 minutes) for 10x100000 and uses 800 MB
35+intmapMatrixRelation :: Int -> Int -> HS.HashSet (IM.IntMap Atom)
36+intmapMatrixRelation attributeCount tupleCount = HS.fromList $ map mapper [0..tupleCount]
37+ where
38+ mapper tupCount = IM.fromList $ map (\c-> (c, IntAtom tupCount)) [0..attributeCount]
39+
40+instance Hash.Hashable (IM.IntMap Atom) where
41+ hashWithSalt salt tupMap = Hash.hashWithSalt salt (show tupMap)
42+
43+vectorMatrixRun = do
44+ let matrix = vectorMatrixRelation 100 100000
45+ putStrLn (show matrix)
46+
47+-- 20 s 90 MBs- a clear win- ideal size is 10 * 100000 * 8 bytes = 80 MB! without IntAtom wrapper
48+--with IntAtom wrapper: 1m12s 90 MB
49+vectorMatrixRelation :: Int -> Int -> HS.HashSet (V.Vector Atom)
50+vectorMatrixRelation attributeCount tupleCount = HS.fromList $ map mapper [0..tupleCount]
51+ where
52+ mapper tupCount = V.replicate attributeCount (IntAtom tupCount)
53+
54+instance Hash.Hashable (V.Vector Atom) where
55+ hashWithSalt salt vec = Hash.hashWithSalt salt (show vec)