git.iko.soy/streamly
on master help sign in
Use Haskell based test driver

Replace bin/test.sh with a Haskell implementation

commit5435aa1b1624e52ff83165da339540cff7f57c41 authorHarendra Kumar <harendra@composewell.com> dateWed 27 Apr 2022, 11:59 (5 years ago) parent326bcda
11 files changed+383−72
compare with master
unifiedsplit
.github/workflows/regression-check.yml+3−3 .packcheck.ignore+2 benchmark/bench-report/bin/bench-runner.sh+1 benchmark/report/Main.hs+2−24 cabal.project.report+1 streamly.cabal+2 targets/Targets.hs+139−45 test/report/Main.hs+34 test/report/cabal.project+1 test/report/default.nix+157 test/report/streamly-test-report.cabal+41
.github/workflows/regression-check.yml+3−3
@@ -120,7 +120,7 @@ jobs:
120120
121121 - name: Run benchmarks and append
122122 run: |
123− cabal run report --project-file=cabal.project.report -- --benchmarks "$CI_BENCHMARKS" --raw --append
123+ cabal run report --project-file=cabal.project.report -- --targets "$CI_BENCHMARKS" --raw --append
124124
125125 # -----------------------------------------------------------------
126126 # -- Compare
@@ -128,7 +128,7 @@ jobs:
128128
129129 - name: List all benchmarks
130130 run: |
131− cabal run report --project-file=cabal.project.report -- --benchmarks "$CI_BENCHMARKS" --no-measure
131+ cabal run report --project-file=cabal.project.report -- --targets "$CI_BENCHMARKS" --no-measure
132132
133133 - name: Compare benchmarks
134134 run: |
@@ -146,7 +146,7 @@ jobs:
146146
147147 echo
148148 echo "Checking $bname for regressions greater than $cutoff percent"
149− ! cabal run report --project-file=cabal.project.report -- --benchmarks "$bname" \
149+ ! cabal run report --project-file=cabal.project.report -- --targets "$bname" \
150150 --fields "$CI_FIELDS" \
151151 --no-measure --silent \
152152 --diff-cutoff-percent $cutoff \
.packcheck.ignore+2
@@ -28,3 +28,5 @@ hie.yaml
2828stack.yaml
2929targets/default.nix
3030targets/cabal.project
31+test/report/default.nix
32+test/report/cabal.project
benchmark/bench-report/bin/bench-runner.sh+1
@@ -265,6 +265,7 @@ $RTS_OPTIONS \
265265 then
266266 # Escape "\" and double quotes in benchmark names
267267 BENCH_NAME_ESC=$(echo "$BENCH_NAME_ORIG" | sed -e 's/\\/\\\\/g' | sed -e 's/"/\\"/g')
268+ echo "$BENCH_NAME_ESC"
268269 $BENCH_EXEC_PATH \
269270 -j 1 \
270271 $RTS_OPTIONS1 \
benchmark/report/Main.hs+2−24
@@ -7,29 +7,7 @@ import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
77import Data.Map (Map)
88
99import qualified Data.Map as Map
10−import Targets
11−
12−comparisons :: Map String [String]
13−comparisons =
14− Map.fromList
15− [ ("base_stream_cmp", ["Data.Stream.StreamD", "Data.Stream.StreamK"])
16− , ("serial_wserial_cmp", ["Prelude.Serial", "Prelude.WSerial"])
17− , ("serial_async_cmp", ["Prelude.Serial", "Prelude.Async"])
18− , ( "concurrent_cmp"
19− , [ "Prelude.Async"
20− , "Prelude.WAsync"
21− , "Prelude.Ahead"
22− , "Prelude.Parallel"
23− ])
24− , ( "array_cmp"
25− , [ "Data.Array.Foreign"
26− , "Data.Array.Prim"
27− , "Data.Array"
28− , "Data.Array.Prim.Pinned"
29− ])
30− , ("pinned_array_cmp", ["Data.Array.Foreign", "Data.Array.Prim.Pinned"])
31− , ("base_parser_cmp", ["Data.Parser.ParserD", "Data.Parser.ParserK"])
32− ]
10+import Targets (targets)
3311
3412rtsOpts :: String -> String -> String
3513rtsOpts exeName benchName0 = unwords [general, exeSpecific, benchSpecific]
@@ -151,4 +129,4 @@ speedOpts exeName benchName0 = exeSpecific <|> benchSpecific
151129 | otherwise = Nothing
152130
153131main :: IO ()
154−main = mainWith groupTargets individualTargets comparisons speedOpts rtsOpts
132+main = mainWith targets speedOpts rtsOpts
cabal.project.report+1
@@ -1,6 +1,7 @@
11packages:
22 targets/streamly-targets.cabal
33 benchmark/report/report.cabal
4+ test/report/streamly-test-report.cabal
45 streamly.cabal
56 core/streamly-core.cabal
67
streamly.cabal+2
@@ -88,6 +88,8 @@ extra-source-files:
8888 docs/streamly-docs.cabal
8989 targets/streamly-targets.cabal
9090 targets/Targets.hs
91+ test/report/Main.hs
92+ test/report/streamly-test-report.cabal
9193
9294 -- This is duplicated
9395 src/Streamly/Internal/Data/Stream/Instances.hs
targets/Targets.hs+139−45
@@ -1,55 +1,149 @@
11module Targets
2− ( groupTargets
3− , individualTargets
2+ ( targets
43 )
54where
65
7−import Data.Map (Map)
6+targets :: [(String, [String])]
7+targets =
8+ [ -- Base streams
9+ ("Data.Stream.StreamD",
10+ [ "base_stream_grp"
11+ , "base_stream_cmp"
12+ , "noTest"
13+ ]
14+ )
15+ , ("Data.Stream.StreamK",
16+ [ "base_stream_grp"
17+ , "base_stream_cmp"
18+ , "noTest"
19+ ]
20+ )
21+ , ("Data.Stream.StreamDK",
22+ [ "base_stream_grp"
23+ , "noTest"
24+ ]
25+ )
826
9−import qualified Data.Map as Map
27+ -- Streams
28+ , ("Prelude.Serial", ["serial_wserial_cmp"])
29+ , ("Prelude.WSerial", ["serial_wserial_cmp"])
30+ , ("Prelude.Serial",
31+ [ "prelude_serial_grp"
32+ , "infinite_grp"
33+ , "serial_wserial_cmp"
34+ , "serial_async_cmp"
35+ ]
36+ )
37+ , ("Prelude.WSerial",
38+ [ "prelude_serial_grp"
39+ , "infinite_grp"
40+ , "serial_wserial_cmp"
41+ ]
42+ )
43+ , ("Prelude.ZipSerial",
44+ [ "prelude_serial_grp"
45+ , "infinite_grp"
46+ ]
47+ )
48+ , ("Prelude.Async",
49+ [ "prelude_concurrent_grp"
50+ , "infinite_grp"
51+ , "concurrent_cmp"
52+ , "serial_async_cmp"
53+ ]
54+ )
55+ , ("Prelude.WAsync",
56+ [ "prelude_concurrent_grp"
57+ , "infinite_grp"
58+ , "concurrent_cmp"
59+ ]
60+ )
61+ , ("Prelude.Ahead",
62+ [ "prelude_concurrent_grp"
63+ , "infinite_grp"
64+ , "concurrent_cmp"
65+ ]
66+ )
67+ , ("Prelude.Parallel",
68+ [ "prelude_concurrent_grp"
69+ , "infinite_grp"
70+ , "concurrent_cmp"
71+ ]
72+ )
73+ , ("Prelude.ZipAsync",
74+ [ "prelude_concurrent_grp"
75+ , "infinite_grp"
76+ ]
77+ )
78+ , ("Prelude.Concurrent", [ "prelude_other_grp" ])
79+ , ("Prelude.Rate",
80+ [ "prelude_other_grp"
81+ , "infinite_grp"
82+ , "testDevOnly"
83+ ]
84+ )
85+ , ("Prelude.Adaptive",
86+ [ "prelude_other_grp"
87+ , "noTest"
88+ ]
89+ )
1090
11−groupTargets :: Map String [String]
12−groupTargets =
13− Map.fromList
14− [ ( "base_stream_grp"
15− , [ "Data.Stream.StreamD"
16− , "Data.Stream.StreamK"
17− , "Data.Stream.StreamDK"
18− ])
19− , ("prelude_serial_grp", preludeSerialGrp)
20− , ("prelude_concurrent_grp", preludeConcurrentGrp)
21− , ( "prelude_other_grp"
22− , ["Prelude.Rate", "Prelude.Concurrent", "Prelude.Adaptive"])
23− , ( "array_grp"
24− , [ "Data.Array"
25− , "Data.Array.Foreign"
26− , "Data.Array.Prim"
27− , "Data.SmallArray"
28− , "Data.Array.Prim.Pinned"
29− ])
30− , ("base_parser_grp", ["Data.Parser.ParserD", "Data.Parser.ParserK"])
31− , ("parser_grp", ["Data.Fold", "Data.Parser"])
32− , ("list_grp ", [])
33− , ( "infinite_grp"
34− , preludeSerialGrp ++ preludeConcurrentGrp ++ ["Prelude.Rate"])
35− ]
91+ -- Arrays
92+ , ("Data.Array",
93+ [ "array_grp"
94+ , "array_cmp"
95+ ]
96+ )
97+ , ("Data.Array.Foreign",
98+ [ "array_grp"
99+ , "array_cmp"
100+ , "pinned_array_cmp"
101+ ]
102+ )
103+ , ("Data.Array.Prim",
104+ [ "array_grp"
105+ , "array_cmp"
106+ ]
107+ )
108+ , ("Data.Array.Prim.Pinned",
109+ [ "array_grp"
110+ , "array_cmp"
111+ , "pinned_array_cmp"
112+ ]
113+ )
114+ , ("Data.SmallArray",
115+ [ "array_grp" ]
116+ )
36117
37− where
118+ -- Parsers
119+ , ("Data.Parser.ParserD",
120+ [ "base_parser_grp"
121+ , "base_parser_cmp"
122+ ]
123+ )
124+ , ("Data.Parser.ParserK",
125+ [ "base_parser_grp"
126+ , "base_parser_cmp"
127+ , "noTest"
128+ ]
129+ )
130+ , ("Data.Fold", [ "parser_grp" ])
131+ , ("Data.Parser", [ "parser_grp" ])
38132
39− preludeSerialGrp = ["Prelude.Serial", "Prelude.WSerial", "Prelude.ZipSerial"]
40− preludeConcurrentGrp =
41− [ "Prelude.Async"
42− , "Prelude.WAsync"
43− , "Prelude.Ahead"
44− , "Prelude.Parallel"
45− , "Prelude.ZipAsync"
46− ]
133+ , ("Data.Unfold", [])
134+ , ("FileSystem.Handle", [])
135+ , ("Unicode.Stream", [])
136+ , ("Unicode.Utf8", ["noTest"])
137+ , ("Unicode.Char", ["testDevOnly"])
47138
48−individualTargets :: [String]
49−individualTargets =
50− [ "Data.Unfold"
51− , "Unicode.Stream"
52− , "Unicode.Char"
53− , "Unicode.Utf8"
54− , "FileSystem.Handle"
139+ -- test only, no benchmarks
140+ , ("Prelude", ["prelude_other_grp", "noBench"])
141+ , ("Prelude.Fold", ["prelude_other_grp", "noBench"])
142+ , ("FileSystem.Event", ["noBench"])
143+ , ("Network.Socket", ["noBench"])
144+ , ("Network.Inet.TCP", ["noBench"])
145+ , ("version-bounds", ["noBench"])
146+
147+ , ("Data.List", ["list_grp", "noBench"])
148+ , ("Data.List.Base", ["list_grp", "noBench"])
55149 ]
test/report/Main.hsnew file+34
@@ -0,0 +1,34 @@
1+module Main where
2+
3+import TestRunner (mainWith)
4+import BuildLib (Quickness(..))
5+import Control.Applicative ((<|>))
6+import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
7+import Data.Map (Map)
8+
9+import qualified Data.Map as Map
10+import Targets (targets)
11+
12+rtsOpts :: Bool -> String -> String -> String
13+rtsOpts relaxMem exeName target =
14+
15+ if relaxMem
16+ then "-K16M -M1024M"
17+ else "-K8M -M64M " <> targetSpecific
18+
19+ where
20+
21+ targetSpecific =
22+ case target of
23+ -- XXX Data.Array.* heap requirement increased for GHC-8.10
24+ "Data.Array.Foreign" -> "-M128M"
25+ "Data.Array.Prim" -> "-M128M"
26+ "Data.Array.Prim.Pinned" -> "-M128M"
27+ "Prelude.Rate" -> "-M512M"
28+ -- For -O0 case writeChunks test fails, maybe we should have a
29+ -- separate flag for O0 case?
30+ "FileSystem.Handle" -> "-K16M"
31+ _ -> ""
32+
33+main :: IO ()
34+main = mainWith targets rtsOpts
test/report/cabal.projectnew file+1
@@ -0,0 +1 @@
1+packages: ., ../../targets
test/report/default.nixnew file+157
@@ -0,0 +1,157 @@
1+let
2+ nixpkgsPath_21_11 =
3+ "https://github.com/NixOS/nixpkgs/archive/refs/tags/21.11.tar.gz";
4+ nixpkgsDefault = import (builtins.fetchTarball nixpkgsPath_21_11) { };
5+in
6+{
7+ nixpkgs ? nixpkgsDefault
8+, compiler ? "default"
9+}:
10+let haskellPackages =
11+ if compiler == "default"
12+ then nixpkgs.haskellPackages
13+ else nixpkgs.haskell.packages.${compiler};
14+
15+ mkPackage = super: pkg: path: opts: inShell:
16+ let orig = super.callCabal2nixWithOptions pkg path opts {};
17+ in if inShell
18+ # Avoid copying the source directory to nix store by using
19+ # src = null.
20+ then orig.overrideAttrs (oldAttrs: { src = null; })
21+ else orig;
22+
23+ mkPkgGit = super: gitSrc: rev: name:
24+ let
25+ src = builtins.fetchGit {
26+ url = gitSrc;
27+ rev = rev;
28+ };
29+ in super.callCabal2nix name src { };
30+
31+ mkPkgGitRef = super: gitSrc: rev: ref: name:
32+ let
33+ src = builtins.fetchGit {
34+ url = gitSrc;
35+ rev = rev;
36+ ref = ref;
37+ };
38+ in super.callCabal2nix name src { };
39+
40+ mkHaskellPackages = inShell:
41+ haskellPackages.override {
42+ overrides = self: super:
43+ with nixpkgs.haskell.lib;
44+ {
45+ report = mkPackage super "streamly-test-report" ./. "" inShell;
46+
47+ bench-report =
48+ nixpkgs.haskell.lib.overrideCabal
49+ (let src = fetchGit {
50+ url = "git@github.com:composewell/bench-report.git";
51+ rev = "c11f79b202d812efe4d1785e0537f1ad4d947bfe";
52+ }; in super.callCabal2nix "bench-report" src {})
53+ (old:
54+ { enableLibraryProfiling = false;
55+ doHaddock = false;
56+ });
57+
58+# streamly-coreutils = let
59+# src = "git@github.com:composewell/streamly-coreutils.git";
60+# rev = "a12756efe24bbf303f2f6d66a95426c7657d297b";
61+# in mkPkgGit super src rev "streamly-coreutils";
62+
63+ streamly-coreutils =
64+ nixpkgs.haskell.lib.overrideCabal
65+ (let src = fetchGit {
66+ url = "git@github.com:composewell/streamly-coreutils.git";
67+ rev = "8fa4a3ebedb3f970f64b7d78d8ab340b0f771abe";
68+ }; in super.callCabal2nix "streamly-coreutils" src {})
69+ (old:
70+ { librarySystemDepends =
71+ if builtins.currentSystem == "x86_64-darwin"
72+ then [nixpkgs.darwin.apple_sdk.frameworks.Cocoa]
73+ else [];
74+ #enableLibraryProfiling = false;
75+ doHaddock = false;
76+ doCheck = false;
77+ });
78+
79+ streamly-shell = let
80+ src = "https://github.com/composewell/streamly-shell.git";
81+ rev = "7d77ddc0c06aec21fbbfd1b920ec4f58da654a1b";
82+ in mkPkgGit super src rev "streamly-shell";
83+
84+ streamly-process = let
85+ src = "https://github.com/composewell/streamly-process.git";
86+ rev = "a56ef61c1208c2d564a526eba315d903c0b4af2c";
87+ in nixpkgs.haskell.lib.dontCheck
88+ (mkPkgGit super src rev "streamly-process");
89+
90+ #streamly-process = nixpkgs.haskell.lib.dontCheck
91+ # (super.callHackageDirect {
92+ # pkg = "streamly-process";
93+ # ver = "0.1.0";
94+ # sha256 = "01nxisqfmn29fbirdsx71sfjp2rdqwrf469qyjcql2d11i1bxn94";
95+ # } { });
96+
97+ # streamly = super.callHackageDirect {
98+ # pkg = "streamly";
99+ # ver = "0.8.0";
100+ # sha256 = "0vy2lkljizlhpbpbybmg9jcmj2g4s1aaqd2dzy5c0y0n4rgwxask";
101+ # } { };
102+ streamly =
103+ nixpkgs.haskell.lib.overrideCabal
104+ #(super.callHackageDirect
105+ # { pkg = "streamly";
106+ # ver = "0.8.2";
107+ # sha256 = "sha256-CjFq9SCdbgLZa7NqOE4OtC8OaFg4vK8VmIDjGU5rGko=";
108+ # } {})
109+ (let src = fetchGit {
110+ url = "git@github.com:composewell/streamly.git";
111+ rev = "a849812d9960d5b7868e770156ab774ee8f70157";
112+ }; in super.callCabal2nix "streamly" src {})
113+ (old:
114+ { librarySystemDepends =
115+ if builtins.currentSystem == "x86_64-darwin"
116+ then [nixpkgs.darwin.apple_sdk.frameworks.Cocoa]
117+ else [];
118+ #enableLibraryProfiling = false;
119+ doHaddock = false;
120+ });
121+
122+ streamly-core =
123+ nixpkgs.haskell.lib.overrideCabal
124+ (let src = fetchGit {
125+ url = "git@github.com:composewell/streamly.git";
126+ rev = "a849812d9960d5b7868e770156ab774ee8f70157";
127+ }; in super.callCabal2nix "streamly-core" "${src}/core" {})
128+ (old:
129+ { librarySystemDepends =
130+ if builtins.currentSystem == "x86_64-darwin"
131+ then [nixpkgs.darwin.apple_sdk.frameworks.Cocoa]
132+ else [];
133+ #enableLibraryProfiling = false;
134+ doHaddock = false;
135+ });
136+
137+ };
138+ };
139+
140+ drv = mkHaskellPackages true;
141+
142+ shell = drv.shellFor {
143+ packages = p:
144+ [ p.report
145+ ];
146+ # Use a better prompt
147+ shellHook = ''
148+ export CABAL_DIR="$(pwd)/.cabal.nix"
149+ if test -n "$PS_SHELL"
150+ then
151+ export PS1="$PS_SHELL\[$bldred\](nix)\[$txtrst\] "
152+ fi
153+ '';
154+ };
155+in if nixpkgs.lib.inNixShell
156+ then shell
157+ else (mkHaskellPackages false).streamly-test-report
test/report/streamly-test-report.cabalnew file+41
@@ -0,0 +1,41 @@
1+cabal-version: 2.4
2+name: streamly-test-report
3+version: 0.1.0.0
4+
5+-- A short (one-line) description of the package.
6+-- synopsis:
7+
8+-- A longer description of the package.
9+-- description:
10+
11+-- A URL where users can report bugs.
12+-- bug-reports:
13+
14+-- The license under which the package is released.
15+-- license:
16+
17+-- The package author(s).
18+-- author:
19+
20+-- An email address to which users can send suggestions, bug reports, and patches.
21+-- maintainer:
22+
23+-- A copyright notice.
24+-- copyright:
25+-- category:
26+-- extra-source-files: CHANGELOG.md
27+
28+executable test-report
29+ main-is: Main.hs
30+
31+ -- Modules included in this executable, other than Main.
32+ -- other-modules:
33+
34+ -- LANGUAGE extensions used by modules in this package.
35+ -- other-extensions:
36+ build-depends: base
37+ , bench-report
38+ , containers
39+ , streamly-targets
40+ hs-source-dirs: .
41+ default-language: Haskell2010