5435aa1 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Use Haskell based test driver

5435aa1Harendra Kumar5 years ago

.github/workflows/regression-check.yml+3−3

compare with master

unifiedsplit

@@ -120,7 +120,7 @@ jobs:

120

121 - name: Run benchmarks and append

122 run: |

123− cabal run report --project-file=cabal.project.report -- --benchmarks "$CI_BENCHMARKS" --raw --append

123+ cabal run report --project-file=cabal.project.report -- --targets "$CI_BENCHMARKS" --raw --append

124

125 # -----------------------------------------------------------------

126 # -- Compare

@@ -128,7 +128,7 @@ jobs:

128

129 - name: List all benchmarks

130 run: |

131− cabal run report --project-file=cabal.project.report -- --benchmarks "$CI_BENCHMARKS" --no-measure

131+ cabal run report --project-file=cabal.project.report -- --targets "$CI_BENCHMARKS" --no-measure

132

133 - name: Compare benchmarks

134 run: |

@@ -146,7 +146,7 @@ jobs:

146

147 echo

148 echo "Checking $bname for regressions greater than $cutoff percent"

149− ! cabal run report --project-file=cabal.project.report -- --benchmarks "$bname" \

149+ ! cabal run report --project-file=cabal.project.report -- --targets "$bname" \

150 --fields "$CI_FIELDS" \

151 --no-measure --silent \

152 --diff-cutoff-percent $cutoff \

.packcheck.ignore+2

@@ -28,3 +28,5 @@ hie.yaml

28stack.yaml

29targets/default.nix

30targets/cabal.project

31+test/report/default.nix

32+test/report/cabal.project

benchmark/bench-report/bin/bench-runner.sh+1

@@ -265,6 +265,7 @@ $RTS_OPTIONS \

265 then

266 # Escape "\" and double quotes in benchmark names

267 BENCH_NAME_ESC=$(echo "$BENCH_NAME_ORIG" | sed -e 's/\\/\\\\/g' | sed -e 's/"/\\"/g')

268+ echo "$BENCH_NAME_ESC"

268 $BENCH_EXEC_PATH \

269 $BENCH_EXEC_PATH \

269 -j 1 \

270 -j 1 \

270 $RTS_OPTIONS1 \

271 $RTS_OPTIONS1 \

benchmark/report/Main.hs+2−24

@@ -7,29 +7,7 @@ import Data.List (isInfixOf, isPrefixOf, isSuffixOf)

7import Data.Map (Map)

9import qualified Data.Map as Map

10−import Targets

10+import Targets (targets)

11−

12−comparisons :: Map String [String]

13−comparisons =

14− Map.fromList

15− [ ("base_stream_cmp", ["Data.Stream.StreamD", "Data.Stream.StreamK"])

16− , ("serial_wserial_cmp", ["Prelude.Serial", "Prelude.WSerial"])

17− , ("serial_async_cmp", ["Prelude.Serial", "Prelude.Async"])

18− , ( "concurrent_cmp"

19− , [ "Prelude.Async"

20− , "Prelude.WAsync"

21− , "Prelude.Ahead"

22− , "Prelude.Parallel"

23− ])

24− , ( "array_cmp"

25− , [ "Data.Array.Foreign"

26− , "Data.Array.Prim"

27− , "Data.Array"

28− , "Data.Array.Prim.Pinned"

29− ])

30− , ("pinned_array_cmp", ["Data.Array.Foreign", "Data.Array.Prim.Pinned"])

31− , ("base_parser_cmp", ["Data.Parser.ParserD", "Data.Parser.ParserK"])

32− ]

34rtsOpts :: String -> String -> String

12rtsOpts :: String -> String -> String

35rtsOpts exeName benchName0 = unwords [general, exeSpecific, benchSpecific]

13rtsOpts exeName benchName0 = unwords [general, exeSpecific, benchSpecific]

@@ -151,4 +129,4 @@ speedOpts exeName benchName0 = exeSpecific <|> benchSpecific

151 | otherwise = Nothing

129 | otherwise = Nothing

152

130

153main :: IO ()

131main :: IO ()

154−main = mainWith groupTargets individualTargets comparisons speedOpts rtsOpts

132+main = mainWith targets speedOpts rtsOpts

cabal.project.report+1

@@ -1,6 +1,7 @@

1packages:

2 targets/streamly-targets.cabal

3 benchmark/report/report.cabal

4+ test/report/streamly-test-report.cabal

4 streamly.cabal

5 streamly.cabal

5 core/streamly-core.cabal

6 core/streamly-core.cabal

streamly.cabal+2

@@ -88,6 +88,8 @@ extra-source-files:

88 docs/streamly-docs.cabal

89 targets/streamly-targets.cabal

90 targets/Targets.hs

91+ test/report/Main.hs

92+ test/report/streamly-test-report.cabal

92 -- This is duplicated

94 -- This is duplicated

93 src/Streamly/Internal/Data/Stream/Instances.hs

95 src/Streamly/Internal/Data/Stream/Instances.hs

targets/Targets.hs+139−45

@@ -1,55 +1,149 @@

1module Targets

2− ( groupTargets

2+ ( targets

3− , individualTargets

4 )

3 )

5where

4where

7−import Data.Map (Map)

6+targets :: [(String, [String])]

7+targets =

8+ [ -- Base streams

9+ ("Data.Stream.StreamD",

10+ [ "base_stream_grp"

11+ , "base_stream_cmp"

12+ , "noTest"

13+ ]

14+ )

15+ , ("Data.Stream.StreamK",

16+ [ "base_stream_grp"

17+ , "base_stream_cmp"

18+ , "noTest"

19+ ]

20+ )

21+ , ("Data.Stream.StreamDK",

22+ [ "base_stream_grp"

23+ , "noTest"

24+ ]

25+ )

9−import qualified Data.Map as Map

27+ -- Streams

28+ , ("Prelude.Serial", ["serial_wserial_cmp"])

29+ , ("Prelude.WSerial", ["serial_wserial_cmp"])

30+ , ("Prelude.Serial",

31+ [ "prelude_serial_grp"

32+ , "infinite_grp"

33+ , "serial_wserial_cmp"

34+ , "serial_async_cmp"

35+ ]

36+ )

37+ , ("Prelude.WSerial",

38+ [ "prelude_serial_grp"

39+ , "infinite_grp"

40+ , "serial_wserial_cmp"

41+ ]

42+ )

43+ , ("Prelude.ZipSerial",

44+ [ "prelude_serial_grp"

45+ , "infinite_grp"

46+ ]

47+ )

48+ , ("Prelude.Async",

49+ [ "prelude_concurrent_grp"

50+ , "infinite_grp"

51+ , "concurrent_cmp"

52+ , "serial_async_cmp"

53+ ]

54+ )

55+ , ("Prelude.WAsync",

56+ [ "prelude_concurrent_grp"

57+ , "infinite_grp"

58+ , "concurrent_cmp"

59+ ]

60+ )

61+ , ("Prelude.Ahead",

62+ [ "prelude_concurrent_grp"

63+ , "infinite_grp"

64+ , "concurrent_cmp"

65+ ]

66+ )

67+ , ("Prelude.Parallel",

68+ [ "prelude_concurrent_grp"

69+ , "infinite_grp"

70+ , "concurrent_cmp"

71+ ]

72+ )

73+ , ("Prelude.ZipAsync",

74+ [ "prelude_concurrent_grp"

75+ , "infinite_grp"

76+ ]

77+ )

78+ , ("Prelude.Concurrent", [ "prelude_other_grp" ])

79+ , ("Prelude.Rate",

80+ [ "prelude_other_grp"

81+ , "infinite_grp"

82+ , "testDevOnly"

83+ ]

84+ )

85+ , ("Prelude.Adaptive",

86+ [ "prelude_other_grp"

87+ , "noTest"

88+ ]

89+ )

11−groupTargets :: Map String [String]

91+ -- Arrays

12−groupTargets =

92+ , ("Data.Array",

13− Map.fromList

93+ [ "array_grp"

14− [ ( "base_stream_grp"

94+ , "array_cmp"

15− , [ "Data.Stream.StreamD"

16− , "Data.Stream.StreamK"

17− , "Data.Stream.StreamDK"

18− ])

95+ ]

19− , ("prelude_serial_grp", preludeSerialGrp)

96+ )

20− , ("prelude_concurrent_grp", preludeConcurrentGrp)

21− , ( "prelude_other_grp"

22− , ["Prelude.Rate", "Prelude.Concurrent", "Prelude.Adaptive"])

23− , ( "array_grp"

24− , [ "Data.Array"

25− , "Data.Array.Foreign"

97+ , ("Data.Array.Foreign",

98+ [ "array_grp"

99+ , "array_cmp"

100+ , "pinned_array_cmp"

101+ ]

102+ )

26− , "Data.Array.Prim"

103+ , ("Data.Array.Prim",

27− , "Data.SmallArray"

104+ [ "array_grp"

105+ , "array_cmp"

106+ ]

107+ )

28− , "Data.Array.Prim.Pinned"

108+ , ("Data.Array.Prim.Pinned",

29− ])

109+ [ "array_grp"

30− , ("base_parser_grp", ["Data.Parser.ParserD", "Data.Parser.ParserK"])

110+ , "array_cmp"

31− , ("parser_grp", ["Data.Fold", "Data.Parser"])

111+ , "pinned_array_cmp"

32− , ("list_grp ", [])

112+ ]

33− , ( "infinite_grp"

113+ )

34− , preludeSerialGrp ++ preludeConcurrentGrp ++ ["Prelude.Rate"])

114+ , ("Data.SmallArray",

35− ]

115+ [ "array_grp" ]

116+ )

117

37− where

118+ -- Parsers

119+ , ("Data.Parser.ParserD",

120+ [ "base_parser_grp"

121+ , "base_parser_cmp"

122+ ]

123+ )

124+ , ("Data.Parser.ParserK",

125+ [ "base_parser_grp"

126+ , "base_parser_cmp"

127+ , "noTest"

128+ ]

129+ )

130+ , ("Data.Fold", [ "parser_grp" ])

131+ , ("Data.Parser", [ "parser_grp" ])

132

39− preludeSerialGrp = ["Prelude.Serial", "Prelude.WSerial", "Prelude.ZipSerial"]

133+ , ("Data.Unfold", [])

40− preludeConcurrentGrp =

134+ , ("FileSystem.Handle", [])

41− [ "Prelude.Async"

135+ , ("Unicode.Stream", [])

42− , "Prelude.WAsync"

136+ , ("Unicode.Utf8", ["noTest"])

43− , "Prelude.Ahead"

137+ , ("Unicode.Char", ["testDevOnly"])

44− , "Prelude.Parallel"

45− , "Prelude.ZipAsync"

46− ]

138

48−individualTargets :: [String]

139+ -- test only, no benchmarks

49−individualTargets =

140+ , ("Prelude", ["prelude_other_grp", "noBench"])

50− [ "Data.Unfold"

141+ , ("Prelude.Fold", ["prelude_other_grp", "noBench"])

51− , "Unicode.Stream"

52− , "Unicode.Char"

53− , "Unicode.Utf8"

54− , "FileSystem.Handle"

142+ , ("FileSystem.Event", ["noBench"])

143+ , ("Network.Socket", ["noBench"])

144+ , ("Network.Inet.TCP", ["noBench"])

145+ , ("version-bounds", ["noBench"])

146+

147+ , ("Data.List", ["list_grp", "noBench"])

148+ , ("Data.List.Base", ["list_grp", "noBench"])

55 ]

149 ]

test/report/Main.hsnew file+34

@@ -0,0 +1,34 @@

1+module Main where

3+import TestRunner (mainWith)

4+import BuildLib (Quickness(..))

5+import Control.Applicative ((<|>))

6+import Data.List (isInfixOf, isPrefixOf, isSuffixOf)

7+import Data.Map (Map)

9+import qualified Data.Map as Map

10+import Targets (targets)

11+

12+rtsOpts :: Bool -> String -> String -> String

13+rtsOpts relaxMem exeName target =

14+

15+ if relaxMem

16+ then "-K16M -M1024M"

17+ else "-K8M -M64M " <> targetSpecific

18+

19+ where

20+

21+ targetSpecific =

22+ case target of

23+ -- XXX Data.Array.* heap requirement increased for GHC-8.10

24+ "Data.Array.Foreign" -> "-M128M"

25+ "Data.Array.Prim" -> "-M128M"

26+ "Data.Array.Prim.Pinned" -> "-M128M"

27+ "Prelude.Rate" -> "-M512M"

28+ -- For -O0 case writeChunks test fails, maybe we should have a

29+ -- separate flag for O0 case?

30+ "FileSystem.Handle" -> "-K16M"

31+ _ -> ""

32+

33+main :: IO ()

34+main = mainWith targets rtsOpts

test/report/cabal.projectnew file+1

@@ -0,0 +1 @@

1+packages: ., ../../targets

test/report/default.nixnew file+157

@@ -0,0 +1,157 @@

1+let

2+ nixpkgsPath_21_11 =

3+ "https://github.com/NixOS/nixpkgs/archive/refs/tags/21.11.tar.gz";

4+ nixpkgsDefault = import (builtins.fetchTarball nixpkgsPath_21_11) { };

5+in

6+{

7+ nixpkgs ? nixpkgsDefault

8+, compiler ? "default"

9+}:

10+let haskellPackages =

11+ if compiler == "default"

12+ then nixpkgs.haskellPackages

13+ else nixpkgs.haskell.packages.${compiler};

14+

15+ mkPackage = super: pkg: path: opts: inShell:

16+ let orig = super.callCabal2nixWithOptions pkg path opts {};

17+ in if inShell

18+ # Avoid copying the source directory to nix store by using

19+ # src = null.

20+ then orig.overrideAttrs (oldAttrs: { src = null; })

21+ else orig;

22+

23+ mkPkgGit = super: gitSrc: rev: name:

24+ let

25+ src = builtins.fetchGit {

26+ url = gitSrc;

27+ rev = rev;

28+ };

29+ in super.callCabal2nix name src { };

30+

31+ mkPkgGitRef = super: gitSrc: rev: ref: name:

32+ let

33+ src = builtins.fetchGit {

34+ url = gitSrc;

35+ rev = rev;

36+ ref = ref;

37+ };

38+ in super.callCabal2nix name src { };

39+

40+ mkHaskellPackages = inShell:

41+ haskellPackages.override {

42+ overrides = self: super:

43+ with nixpkgs.haskell.lib;

44+ {

45+ report = mkPackage super "streamly-test-report" ./. "" inShell;

46+

47+ bench-report =

48+ nixpkgs.haskell.lib.overrideCabal

49+ (let src = fetchGit {

50+ url = "git@github.com:composewell/bench-report.git";

51+ rev = "c11f79b202d812efe4d1785e0537f1ad4d947bfe";

52+ }; in super.callCabal2nix "bench-report" src {})

53+ (old:

54+ { enableLibraryProfiling = false;

55+ doHaddock = false;

56+ });

57+

58+# streamly-coreutils = let

59+# src = "git@github.com:composewell/streamly-coreutils.git";

60+# rev = "a12756efe24bbf303f2f6d66a95426c7657d297b";

61+# in mkPkgGit super src rev "streamly-coreutils";

62+

63+ streamly-coreutils =

64+ nixpkgs.haskell.lib.overrideCabal

65+ (let src = fetchGit {

66+ url = "git@github.com:composewell/streamly-coreutils.git";

67+ rev = "8fa4a3ebedb3f970f64b7d78d8ab340b0f771abe";

68+ }; in super.callCabal2nix "streamly-coreutils" src {})

69+ (old:

70+ { librarySystemDepends =

71+ if builtins.currentSystem == "x86_64-darwin"

72+ then [nixpkgs.darwin.apple_sdk.frameworks.Cocoa]

73+ else [];

74+ #enableLibraryProfiling = false;

75+ doHaddock = false;

76+ doCheck = false;

77+ });

78+

79+ streamly-shell = let

80+ src = "https://github.com/composewell/streamly-shell.git";

81+ rev = "7d77ddc0c06aec21fbbfd1b920ec4f58da654a1b";

82+ in mkPkgGit super src rev "streamly-shell";

83+

84+ streamly-process = let

85+ src = "https://github.com/composewell/streamly-process.git";

86+ rev = "a56ef61c1208c2d564a526eba315d903c0b4af2c";

87+ in nixpkgs.haskell.lib.dontCheck

88+ (mkPkgGit super src rev "streamly-process");

89+

90+ #streamly-process = nixpkgs.haskell.lib.dontCheck

91+ # (super.callHackageDirect {

92+ # pkg = "streamly-process";

93+ # ver = "0.1.0";

94+ # sha256 = "01nxisqfmn29fbirdsx71sfjp2rdqwrf469qyjcql2d11i1bxn94";

95+ # } { });

96+

97+ # streamly = super.callHackageDirect {

98+ # pkg = "streamly";

99+ # ver = "0.8.0";

100+ # sha256 = "0vy2lkljizlhpbpbybmg9jcmj2g4s1aaqd2dzy5c0y0n4rgwxask";

101+ # } { };

102+ streamly =

103+ nixpkgs.haskell.lib.overrideCabal

104+ #(super.callHackageDirect

105+ # { pkg = "streamly";

106+ # ver = "0.8.2";

107+ # sha256 = "sha256-CjFq9SCdbgLZa7NqOE4OtC8OaFg4vK8VmIDjGU5rGko=";

108+ # } {})

109+ (let src = fetchGit {

110+ url = "git@github.com:composewell/streamly.git";

111+ rev = "a849812d9960d5b7868e770156ab774ee8f70157";

112+ }; in super.callCabal2nix "streamly" src {})

113+ (old:

114+ { librarySystemDepends =

115+ if builtins.currentSystem == "x86_64-darwin"

116+ then [nixpkgs.darwin.apple_sdk.frameworks.Cocoa]

117+ else [];

118+ #enableLibraryProfiling = false;

119+ doHaddock = false;

120+ });

121+

122+ streamly-core =

123+ nixpkgs.haskell.lib.overrideCabal

124+ (let src = fetchGit {

125+ url = "git@github.com:composewell/streamly.git";

126+ rev = "a849812d9960d5b7868e770156ab774ee8f70157";

127+ }; in super.callCabal2nix "streamly-core" "${src}/core" {})

128+ (old:

129+ { librarySystemDepends =

130+ if builtins.currentSystem == "x86_64-darwin"

131+ then [nixpkgs.darwin.apple_sdk.frameworks.Cocoa]

132+ else [];

133+ #enableLibraryProfiling = false;

134+ doHaddock = false;

135+ });

136+

137+ };

138+ };

139+

140+ drv = mkHaskellPackages true;

141+

142+ shell = drv.shellFor {

143+ packages = p:

144+ [ p.report

145+ ];

146+ # Use a better prompt

147+ shellHook = ''

148+ export CABAL_DIR="$(pwd)/.cabal.nix"

149+ if test -n "$PS_SHELL"

150+ then

151+ export PS1="$PS_SHELL\[$bldred\](nix)\[$txtrst\] "

152+ fi

153+ '';

154+ };

155+in if nixpkgs.lib.inNixShell

156+ then shell

157+ else (mkHaskellPackages false).streamly-test-report

test/report/streamly-test-report.cabalnew file+41

@@ -0,0 +1,41 @@

1+cabal-version: 2.4

2+name: streamly-test-report

3+version: 0.1.0.0

5+-- A short (one-line) description of the package.

6+-- synopsis:

8+-- A longer description of the package.

9+-- description:

10+

11+-- A URL where users can report bugs.

12+-- bug-reports:

13+

14+-- The license under which the package is released.

15+-- license:

16+

17+-- The package author(s).

18+-- author:

19+

20+-- An email address to which users can send suggestions, bug reports, and patches.

21+-- maintainer:

22+

23+-- A copyright notice.

24+-- copyright:

25+-- category:

26+-- extra-source-files: CHANGELOG.md

27+

28+executable test-report

29+ main-is: Main.hs

30+

31+ -- Modules included in this executable, other than Main.

32+ -- other-modules:

33+

34+ -- LANGUAGE extensions used by modules in this package.

35+ -- other-extensions:

36+ build-depends: base

37+ , bench-report

38+ , containers

39+ , streamly-targets

40+ hs-source-dirs: .

41+ default-language: Haskell2010