master..cb2eb0d · grenade

.gitignore+1−3

@@ -1,6 +1,4 @@

1−cabal.project.local

21.cabal-sandbox/

32cabal.sandbox.config

43dist/

5−dist-newstyle/

6−.stack-work/

4+

.travis.yml+11−12

@@ -2,30 +2,29 @@

22

33# The following enables several GHC versions to be tested; often it's enough to test only against the last release in a major GHC version. Feel free to omit lines listings versions you don't need/want testing for.

44env:

5− - GHCVER=8.0.2

6− - GHCVER=8.2.2

7− - GHCVER=8.4.4

8− - GHCVER=8.6.5

9− - GHCVER=8.8.3

10− - GHCVER=8.10.1

5+ - CABALVER=1.22 GHCVER=7.10.3

6+ - CABALVER=1.24 GHCVER=8.0.2

7+ - CABALVER=2.0 GHCVER=8.2.1

118

129# Note: the distinction between `before_install` and `install` is not important.

1310before_install:

1411 - travis_retry sudo add-apt-repository -y ppa:hvr/ghc

1512 - travis_retry sudo apt-get update

16− - travis_retry sudo apt-get install cabal-install-3.2 ghc-$GHCVER libblas-dev liblapack-dev

17− - export PATH=/opt/cabal/bin:/opt/ghc/$GHCVER/bin:$PATH

13+ - travis_retry sudo apt-get install cabal-install-$CABALVER ghc-$GHCVER happy-1.19.5 alex-3.1.7 libblas-dev liblapack-dev

14+ - export PATH=/opt/alex/3.1.7/bin:/opt/happy/1.19.5/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$HOME/.cabal/bin:$PATH

1815

1916install:

17+ - cabal --version

2018 - echo "$(ghc --version) [$(ghc --print-project-git-commit-id 2> /dev/null || echo '?')]"

21− - travis_retry cabal-3.2 update

19+ - travis_retry cabal update

20+ - cabal sandbox init

21+ - cabal install --enable-tests --enable-benchmarks

2222

2323notifications:

2424 email: false

2525

2626# Here starts the actual work to be performed for the package under test; any command which exits with a non-zero exit code causes the build to fail.

2727script:

28− - cabal-3.2 configure --enable-tests

29− - cabal-3.2 build all

30− - cabal-3.2 test --test-show-details=direct -j1

28+ - cabal build

29+ - cabal test --show-details=streaming

3130

README.md+2−2

@@ -24,7 +24,7 @@ specified and initialised with random weights in a few lines of code with

2424type MNIST

2525 = Network

2626 '[ Convolution 1 10 5 5 1 1, Pooling 2 2 2 2, Relu

27− , Convolution 10 16 5 5 1 1, Pooling 2 2 2 2, Reshape, Relu

27+ , Convolution 10 16 5 5 1 1, Pooling 2 2 2 2, FlattenLayer, Relu

2828 , FullyConnected 256 80, Logit, FullyConnected 80 10, Logit]

2929 '[ 'D2 28 28, 'D3 24 24 10, 'D3 12 12 10, 'D3 12 12 10

3030 , 'D3 8 8 16, 'D3 4 4 16, 'D1 256, 'D1 256

@@ -159,7 +159,7 @@ and the tests run using:

159159./mafia test

160160```

161161

162−Grenade builds with ghc 7.10, 8.0, 8.2 and 8.4.

162+Grenade builds with ghc 7.10 and 8.0.

163163

164164Thanks

165165------

examples/grenade-examples.cabal+9−10

@@ -26,7 +26,7 @@ executable feedforward

2626 , bytestring

2727 , cereal

2828 , either

29− , optparse-applicative >= 0.13 && < 0.18

29+ , optparse-applicative >= 0.13 && < 0.15

3030 , text == 1.2.*

3131 , mtl >= 2.2.1 && < 2.3

3232 , hmatrix

@@ -42,10 +42,10 @@ executable mnist

4242 , grenade

4343 , attoparsec

4444 , either

45− , optparse-applicative >= 0.13 && < 0.18

45+ , optparse-applicative >= 0.13 && < 0.15

4646 , text == 1.2.*

4747 , mtl >= 2.2.1 && < 2.3

48− , hmatrix >= 0.18 && < 0.21

48+ , hmatrix >= 0.18 && < 0.19

4949 , transformers

5050 , semigroups

5151 , singletons

@@ -61,10 +61,10 @@ executable gan-mnist

6161 , bytestring

6262 , cereal

6363 , either

64− , optparse-applicative >= 0.13 && < 0.18

64+ , optparse-applicative >= 0.13 && < 0.15

6565 , text == 1.2.*

6666 , mtl >= 2.2.1 && < 2.3

67− , hmatrix >= 0.18 && < 0.21

67+ , hmatrix >= 0.18 && < 0.19

6868 , transformers

6969 , semigroups

7070 , singletons

@@ -78,10 +78,10 @@ executable recurrent

7878 , grenade

7979 , attoparsec

8080 , either

81− , optparse-applicative >= 0.13 && < 0.18

81+ , optparse-applicative >= 0.13 && < 0.15

8282 , text == 1.2.*

8383 , mtl >= 2.2.1 && < 2.3

84− , hmatrix >= 0.18 && < 0.21

84+ , hmatrix >= 0.18 && < 0.19

8585 , transformers

8686 , semigroups

8787 , singletons

@@ -96,14 +96,13 @@ executable shakespeare

9696 , bytestring

9797 , cereal

9898 , either

99− , optparse-applicative >= 0.13 && < 0.18

99+ , optparse-applicative >= 0.13 && < 0.15

100100 , text == 1.2.*

101101 , mtl >= 2.2.1 && < 2.3

102− , hmatrix >= 0.18 && < 0.21

102+ , hmatrix >= 0.18 && < 0.19

103103 , transformers

104104 , semigroups

105105 , singletons

106− , singletons-base

107106 , vector

108107 , MonadRandom

109108 , containers

examples/mafiadeleted−62

@@ -1,62 +0,0 @@

1−#!/bin/sh -eu

2−

3−: ${MAFIA_HOME:=$HOME/.mafia}

4−: ${MAFIA_VERSIONS:=$MAFIA_HOME/versions}

5−

6−latest_version () {

7− git ls-remote https://github.com/haskell-mafia/mafia | grep refs/heads/master | cut -f 1

8−}

9−

10−build_version() {

11− MAFIA_VERSION="$1"

12− MAFIA_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t 'exec_mafia')

13− MAFIA_FILE=mafia-$MAFIA_VERSION

14− MAFIA_PATH=$MAFIA_VERSIONS/$MAFIA_FILE

15− mkdir -p $MAFIA_VERSIONS

16− echo "Building $MAFIA_FILE in $MAFIA_TEMP"

17− git clone https://github.com/haskell-mafia/mafia $MAFIA_TEMP

18− git --git-dir="$MAFIA_TEMP/.git" --work-tree="$MAFIA_TEMP" reset --hard $MAFIA_VERSION || {

19− echo "mafia version ($MAFIA_VERSION) could not be found." >&2

20− exit 1

21− }

22− (cd "$MAFIA_TEMP" && ./bin/bootstrap) || {

23− got=$?

24− echo "mafia version ($MAFIA_VERSION) could not be built." >&2

25− exit "$got"

26− }

27− chmod +x "$MAFIA_TEMP/.cabal-sandbox/bin/mafia"

28− # Ensure executable is on same file-system so final mv is atomic.

29− mv -f "$MAFIA_TEMP/.cabal-sandbox/bin/mafia" "$MAFIA_PATH.$$"

30− mv "$MAFIA_PATH.$$" "$MAFIA_PATH" || {

31− rm -f "$MAFIA_PATH.$$"

32− echo "INFO: mafia version ($MAFIA_VERSION) already exists not overiding," >&2

33− echo "INFO: this is expected if parallel builds of the same version of" >&2

34− echo "INFO: mafia occur, we are playing by first in, wins." >&2

35− exit 0

36− }

37−}

38−

39−enable_version() {

40− if [ $# -eq 0 ]; then

41− MAFIA_VERSION="$(latest_version)"

42− echo "INFO: No explicit mafia version requested installing latest ($MAFIA_VERSION)." >&2

43− else

44− MAFIA_VERSION="$1"

45− fi

46− [ -x "$MAFIA_HOME/versions/mafia-$MAFIA_VERSION" ] || build_version "$MAFIA_VERSION"

47− ln -sf "$MAFIA_HOME/versions/mafia-$MAFIA_VERSION" "$MAFIA_HOME/versions/mafia"

48−}

49−

50−exec_mafia () {

51− [ -x "$MAFIA_HOME/versions/mafia" ] || enable_version

52− "$MAFIA_HOME/versions/mafia" "$@"

53−}

54−

55−#

56−# The actual start of the script.....

57−#

58−

59−case "${1:-}" in

60−upgrade) shift; enable_version "$@" ;;

61−*) exec_mafia "$@"

62−esac

examples/main/feedforward.hs+1−3

@@ -1,5 +1,4 @@

11{-# LANGUAGE BangPatterns #-}

2−{-# LANGUAGE CPP #-}

32{-# LANGUAGE DataKinds #-}

43{-# LANGUAGE ScopedTypeVariables #-}

54{-# LANGUAGE TypeOperators #-}

@@ -11,9 +10,8 @@ import Data.List ( foldl' )

1110

1211import qualified Data.ByteString as B

1312import Data.Serialize

14−#if ! MIN_VERSION_base(4,13,0)

1513import Data.Semigroup ( (<>) )

16−#endif

14+

1715import GHC.TypeLits

1816

1917import qualified Numeric.LinearAlgebra.Static as SA

examples/main/gan-mnist.hs−3

@@ -1,5 +1,4 @@

11{-# LANGUAGE BangPatterns #-}

2−{-# LANGUAGE CPP #-}

32{-# LANGUAGE DataKinds #-}

43{-# LANGUAGE GADTs #-}

54{-# LANGUAGE ScopedTypeVariables #-}

@@ -44,9 +43,7 @@ import Control.Monad.Trans.Except

4443import qualified Data.Attoparsec.Text as A

4544import qualified Data.ByteString as B

4645import Data.List ( foldl' )

47−#if ! MIN_VERSION_base(4,13,0)

4846import Data.Semigroup ( (<>) )

49−#endif

5047import Data.Serialize

5148import qualified Data.Text as T

5249import qualified Data.Text.IO as T

examples/main/mnist.hs−4

@@ -1,5 +1,4 @@

11{-# LANGUAGE BangPatterns #-}

2−{-# LANGUAGE CPP #-}

32{-# LANGUAGE DataKinds #-}

43{-# LANGUAGE ScopedTypeVariables #-}

54{-# LANGUAGE TypeOperators #-}

@@ -14,10 +13,7 @@ import Control.Monad.Trans.Except

1413

1514import qualified Data.Attoparsec.Text as A

1615import Data.List ( foldl' )

17−#if ! MIN_VERSION_base(4,13,0)

1816import Data.Semigroup ( (<>) )

19−#endif

20−

2117import qualified Data.Text as T

2218import qualified Data.Text.IO as T

2319import qualified Data.Vector.Storable as V

examples/main/recurrent.hs+5−5

@@ -9,12 +9,12 @@

99import Control.Monad ( foldM )

1010import Control.Monad.Random ( MonadRandom, getRandomR )

1111

12−#if MIN_VERSION_base(4,13,0)

12+#if __GLASGOW_HASKELL__ < 800

1313import Data.List ( unfoldr )

1414#else

1515import Data.List ( cycle, unfoldr )

16−import Data.Semigroup ( (<>) )

1716#endif

17+import Data.Semigroup ( (<>) )

1818

1919import qualified Numeric.LinearAlgebra.Static as SA

2020

@@ -31,10 +31,10 @@ import Grenade.Recurrent

3131-- creation function know how to treat the layers.

3232type R = Recurrent

3333

34−type RecNet = RecurrentNetwork '[ R (LSTM 1 1)]

35− '[ 'D1 1, 'D1 1]

34+type RecNet = RecurrentNetwork '[ R (LSTM 1 4), R (LSTM 4 1)]

35+ '[ 'D1 1, 'D1 4, 'D1 1 ]

3636

37−type RecInput = RecurrentInputs '[ R (LSTM 1 1)]

37+type RecInput = RecurrentInputs '[ R (LSTM 1 4), R (LSTM 4 1)]

3838

3939randomNet :: MonadRandom m => m RecNet

4040randomNet = randomRecurrent

examples/main/shakespeare.hs+17−20

@@ -1,5 +1,5 @@

11{-# LANGUAGE BangPatterns #-}

2−{-# LANGUAGE CPP #-}

2+{-# LANGUAGE RecordWildCards #-}

33{-# LANGUAGE DataKinds #-}

44{-# LANGUAGE ScopedTypeVariables #-}

55{-# LANGUAGE TypeOperators #-}

@@ -13,16 +13,18 @@ import Control.Monad.Trans.Except

1313import Data.Char ( isUpper, toUpper, toLower )

1414import Data.List ( foldl' )

1515import Data.Maybe ( fromMaybe )

16−

16+import Data.Semigroup ( (<>) )

1717

1818import qualified Data.Vector as V

1919import Data.Vector ( Vector )

2020

2121import qualified Data.Map as M

22+import Data.Proxy ( Proxy (..) )

2223

2324import qualified Data.ByteString as B

2425import Data.Serialize

2526

27+import Data.Singletons.Prelude

2628import GHC.TypeLits

2729

2830import Numeric.LinearAlgebra.Static ( konst )

@@ -34,16 +36,11 @@ import Grenade.Recurrent

3436import Grenade.Utils.OneHot

3537

3638import System.IO.Unsafe ( unsafeInterleaveIO )

37−import Data.Proxy

38−import Prelude.Singletons

3939

4040-- The defininition for our natural language recurrent network.

4141-- This network is able to learn and generate simple words in

4242-- about an hour.

4343--

44−-- Grab the input from

45−-- https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt

46−--

4744-- This is a first class recurrent net.

4845--

4946-- The F and R types are tagging types to ensure that the runner and

@@ -72,39 +69,39 @@ loadShakespeare :: FilePath -> ExceptT String IO (Vector Int, M.Map Char Int, Ve

7269loadShakespeare path = do

7370 contents <- lift $ readFile path

7471 let annotated = annotateCapitals contents

75− (m,cs) <- ExceptT . return $ hotMap (Proxy :: Proxy 40) annotated

72+ (m,cs) <- ExceptT . return . note "Couldn't fit data in hotMap" $ hotMap (Proxy :: Proxy 40) annotated

7673 hot <- ExceptT . return . note "Couldn't generate hot values" $ traverse (`M.lookup` m) annotated

7774 return (V.fromList hot, m, cs)

7875

7976trainSlice :: LearningParameters -> Shakespeare -> Shakespearian -> Vector Int -> Int -> Int -> (Shakespeare, Shakespearian)

80−trainSlice !lrate !net !recIns input offset size =

77+trainSlice !rate !net !recIns input offset size =

8178 let e = fmap (x . oneHot) . V.toList $ V.slice offset size input

8279 in case reverse e of

8380 (o : l : xs) ->

8481 let examples = reverse $ (l, Just o) : ((,Nothing) <$> xs)

85− in trainRecurrent lrate net recIns examples

82+ in trainRecurrent rate net recIns examples

8683 _ -> error "Not enough input"

8784 where

8885 x = fromMaybe (error "Hot variable didn't fit.")

8986

9087runShakespeare :: ShakespeareOpts -> ExceptT String IO ()

91−runShakespeare opts = do

92− (shakespeare, oneHotMap, oneHotDictionary) <- loadShakespeare $ trainingFile opts

88+runShakespeare ShakespeareOpts {..} = do

89+ (shakespeare, oneHotMap, oneHotDictionary) <- loadShakespeare trainingFile

9390 (net0, i0) <- lift $

94− case loadPath opts of

91+ case loadPath of

9592 Just loadFile -> netLoad loadFile

9693 Nothing -> (,0) <$> randomNet

9794

9895 (trained, bestInput) <- lift $ foldM (\(!net, !io) size -> do

99− xs <- take (iterations opts `div` 10) <$> getRandomRs (0, length shakespeare - size - 1)

100− let (!trained, !bestInput) = foldl' (\(!n, !i) offset -> trainSlice (rate opts) n i shakespeare offset size) (net, io) xs

101− results <- take 1000 <$> generateParagraph trained bestInput (temperature opts) oneHotMap oneHotDictionary ( S1D $ konst 0)

96+ xs <- take (iterations `div` 10) <$> getRandomRs (0, length shakespeare - size - 1)

97+ let (!trained, !bestInput) = foldl' (\(!n, !i) offset -> trainSlice rate n i shakespeare offset size) (net, io) xs

98+ results <- take 1000 <$> generateParagraph trained bestInput temperature oneHotMap oneHotDictionary ( S1D $ konst 0)

10299 putStrLn ("TRAINING STEP WITH SIZE: " ++ show size)

103100 putStrLn (unAnnotateCapitals results)

104101 return (trained, bestInput)

105− ) (net0, i0) $ replicate 10 (sequenceSize opts)

102+ ) (net0, i0) $ replicate 10 sequenceSize

106103

107− case savePath opts of

104+ case savePath of

108105 Just saveFile -> lift . B.writeFile saveFile $ runPut (put trained >> put bestInput)

109106 Nothing -> return ()

110107

@@ -116,12 +113,12 @@ generateParagraph :: forall layers shapes n a. (Last shapes ~ 'D1 n, Head shapes

116113 -> Vector a

117114 -> S ('D1 n)

118115 -> IO [a]

119−generateParagraph n s temp hotmap hotdict =

116+generateParagraph n s temperature hotmap hotdict =

120117 go s

121118 where

122119 go x y =

123120 do let (_, ns, o) = runRecurrent n x y

124− un <- sample temp hotdict o

121+ un <- sample temperature hotdict o

125122 Just re <- return $ makeHot hotmap un

126123 rest <- unsafeInterleaveIO $ go ns re

127124 return (un : rest)

framework/mafiadeleted−62

@@ -1,62 +0,0 @@

1−#!/bin/sh -eu

2−

3−: ${MAFIA_HOME:=$HOME/.mafia}

4−: ${MAFIA_VERSIONS:=$MAFIA_HOME/versions}

5−

6−latest_version () {

7− git ls-remote https://github.com/haskell-mafia/mafia | grep refs/heads/master | cut -f 1

8−}

9−

10−build_version() {

11− MAFIA_VERSION="$1"

12− MAFIA_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t 'exec_mafia')

13− MAFIA_FILE=mafia-$MAFIA_VERSION

14− MAFIA_PATH=$MAFIA_VERSIONS/$MAFIA_FILE

15− mkdir -p $MAFIA_VERSIONS

16− echo "Building $MAFIA_FILE in $MAFIA_TEMP"

17− git clone https://github.com/haskell-mafia/mafia $MAFIA_TEMP

18− git --git-dir="$MAFIA_TEMP/.git" --work-tree="$MAFIA_TEMP" reset --hard $MAFIA_VERSION || {

19− echo "mafia version ($MAFIA_VERSION) could not be found." >&2

20− exit 1

21− }

22− (cd "$MAFIA_TEMP" && ./bin/bootstrap) || {

23− got=$?

24− echo "mafia version ($MAFIA_VERSION) could not be built." >&2

25− exit "$got"

26− }

27− chmod +x "$MAFIA_TEMP/.cabal-sandbox/bin/mafia"

28− # Ensure executable is on same file-system so final mv is atomic.

29− mv -f "$MAFIA_TEMP/.cabal-sandbox/bin/mafia" "$MAFIA_PATH.$$"

30− mv "$MAFIA_PATH.$$" "$MAFIA_PATH" || {

31− rm -f "$MAFIA_PATH.$$"

32− echo "INFO: mafia version ($MAFIA_VERSION) already exists not overiding," >&2

33− echo "INFO: this is expected if parallel builds of the same version of" >&2

34− echo "INFO: mafia occur, we are playing by first in, wins." >&2

35− exit 0

36− }

37−}

38−

39−enable_version() {

40− if [ $# -eq 0 ]; then

41− MAFIA_VERSION="$(latest_version)"

42− echo "INFO: No explicit mafia version requested installing latest ($MAFIA_VERSION)." >&2

43− else

44− MAFIA_VERSION="$1"

45− fi

46− [ -x "$MAFIA_HOME/versions/mafia-$MAFIA_VERSION" ] || build_version "$MAFIA_VERSION"

47− ln -sf "$MAFIA_HOME/versions/mafia-$MAFIA_VERSION" "$MAFIA_HOME/versions/mafia"

48−}

49−

50−exec_mafia () {

51− [ -x "$MAFIA_HOME/versions/mafia" ] || enable_version

52− "$MAFIA_HOME/versions/mafia" "$@"

53−}

54−

55−#

56−# The actual start of the script.....

57−#

58−

59−case "${1:-}" in

60−upgrade) shift; enable_version "$@" ;;

61−*) exec_mafia "$@"

62−esac

grenade.cabal+15−33

@@ -7,7 +7,7 @@ maintainer: Huw Campbell <huw.campbell@gmail.com>

88synopsis: Practical Deep Learning in Haskell

99category: AI, Machine Learning

10−cabal-version: >= 1.10

10+cabal-version: >= 1.8

1111build-type: Simple

1212description:

1313 Grenade is a composable, dependently typed, practical, and fast

@@ -38,17 +38,14 @@ source-repository head

3838library

3939 build-depends:

4040 base >= 4.8 && < 5

41− , bytestring >= 0.10.0

42− , containers >= 0.5 && < 0.7

41+ , bytestring == 0.10.*

42+ , containers >= 0.5 && < 0.6

4343 , cereal >= 0.5 && < 0.6

4444 , deepseq >= 1.4 && < 1.5

45− , hmatrix >= 0.18 && < 0.21

45+ , hmatrix == 0.18.*

4646 , MonadRandom >= 0.4 && < 0.6

47− , primitive >= 0.6 && < 0.8

48− -- Versions of singletons are *tightly* coupled with the

49− -- GHC version so its fine to drop version bounds.

50− , singletons

51− , singletons-base

47+ , primitive >= 0.6 && < 0.7

48+ , singletons >= 2.1 && < 2.4

5249 , vector >= 0.11 && < 0.13

5350

5451 ghc-options:

@@ -56,14 +53,9 @@ library

5653 hs-source-dirs:

5754 src

5855

59− default-language: Haskell2010

60−

6156 if impl(ghc < 8.0)

6257 ghc-options: -fno-warn-incomplete-patterns

63− cpp-options: -DType=*

6458

65− if impl(ghc >= 8.6)

66− default-extensions: NoStarIsType

6759

6860 exposed-modules:

6961 Grenade

@@ -89,7 +81,6 @@ library

8981 Grenade.Layers.Pooling

9082 Grenade.Layers.Relu

9183 Grenade.Layers.Reshape

92− Grenade.Layers.Sinusoid

9384 Grenade.Layers.Softmax

9485 Grenade.Layers.Tanh

9586 Grenade.Layers.Trivial

@@ -132,7 +123,6 @@ test-suite test

132123 hs-source-dirs:

133124 test

134125

135− default-language: Haskell2010

136126

137127 other-modules: Test.Hedgehog.Compat

138128 Test.Hedgehog.Hmatrix

@@ -153,20 +143,16 @@ test-suite test

153143

154144 if impl(ghc < 8.0)

155145 ghc-options: -fno-warn-incomplete-patterns

156− cpp-options: -DType=*

157−

158− if impl(ghc >= 8.6)

159− default-extensions: NoStarIsType

160146

161147 build-depends:

162− base

148+ base >= 4.8 && < 5

163149 , grenade

164− , hedgehog >= 1.0 && < 1.1

150+ , hedgehog >= 0.5 && < 0.6

165151 , hmatrix

166152 , mtl

167153 , singletons

168154 , text == 1.2.*

169− , typelits-witnesses < 0.4

155+ , typelits-witnesses

170156 , transformers

171157 , constraints

172158 , MonadRandom

@@ -186,12 +172,10 @@ benchmark bench

186172 hs-source-dirs:

187173 bench

188174

189− default-language: Haskell2010

190−

191175 build-depends:

192− base

193− , bytestring

194− , criterion >= 1.1 && < 1.6

176+ base >= 3 && < 5

177+ , bytestring == 0.10.*

178+ , criterion >= 1.1 && < 1.3

195179 , grenade

196180 , hmatrix

197181

@@ -205,11 +189,9 @@ benchmark bench-lstm

205189 hs-source-dirs:

206190 bench

207191

208− default-language: Haskell2010

209−

210192 build-depends:

211− base

212− , bytestring

213− , criterion

193+ base >= 3 && < 5

194+ , bytestring == 0.10.*

195+ , criterion == 1.1.*

214196 , grenade

215197 , hmatrix

mafiamode 120000 → 100755+123−1

@@ -1 +0,0 @@

1−framework/mafia

@@ -0,0 +1,123 @@

1+#!/bin/sh -eu

2+

3+: ${MAFIA_HOME:=$HOME/.mafia}

4+

5+fetch_latest () {

6+ if [ -z ${MAFIA_TEST_MODE+x} ]; then

7+ TZ=$(date +"%T")

8+ curl --silent "https://raw.githubusercontent.com/ambiata/mafia/master/script/mafia?$TZ"

9+ else

10+ cat ../script/mafia

11+ fi

12+}

13+

14+latest_version () {

15+ git ls-remote https://github.com/ambiata/mafia | grep refs/heads/master | cut -f 1

16+}

17+

18+local_version () {

19+ awk '/^# Version: / { print $3; exit 0; }' $0

20+}

21+

22+run_upgrade () {

23+ MAFIA_TEMP=$(mktemp 2>/dev/null || mktemp -t 'upgrade_mafia')

24+

25+ clean_up () {

26+ rm -f "$MAFIA_TEMP"

27+ }

28+

29+ trap clean_up EXIT

30+

31+ MAFIA_CUR="$0"

32+

33+ if [ -L "$MAFIA_CUR" ]; then

34+ echo 'Refusing to overwrite a symlink; run `upgrade` from the canonical path.' >&2

35+ exit 1

36+ fi

37+

38+ echo "Checking for a new version of mafia ..."

39+ fetch_latest > $MAFIA_TEMP

40+

41+ LATEST_VERSION=$(latest_version)

42+ echo "# Version: $LATEST_VERSION" >> $MAFIA_TEMP

43+

44+ if ! cmp $MAFIA_CUR $MAFIA_TEMP >/dev/null 2>&1; then

45+ mv $MAFIA_TEMP $MAFIA_CUR

46+ chmod +x $MAFIA_CUR

47+ echo "New version found and upgraded. You can now commit it to your git repo."

48+ else

49+ echo "You have latest mafia."

50+ fi

51+}

52+

53+exec_mafia () {

54+ MAFIA_VERSION=$(local_version)

55+

56+ if [ "x$MAFIA_VERSION" = "x" ]; then

57+ # If we can't find the mafia version, then we need to upgrade the script.

58+ run_upgrade

59+ else

60+ MAFIA_BIN=$MAFIA_HOME/bin

61+ MAFIA_FILE=mafia-$MAFIA_VERSION

62+ MAFIA_PATH=$MAFIA_BIN/$MAFIA_FILE

63+

64+ [ -f "$MAFIA_PATH" ] || {

65+ # Create a temporary directory which will be deleted when the script

66+ # terminates. Unfortunately `mktemp` doesn't behave the same on

67+ # Linux and OS/X so we need to try two different approaches.

68+ MAFIA_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t 'exec_mafia')

69+

70+ # Create a temporary file in MAFIA_BIN so we can do an atomic copy/move dance.

71+ mkdir -p $MAFIA_BIN

72+

73+ clean_up () {

74+ rm -rf "$MAFIA_TEMP"

75+ }

76+

77+ trap clean_up EXIT

78+

79+ echo "Building $MAFIA_FILE in $MAFIA_TEMP"

80+

81+ ( cd "$MAFIA_TEMP"

82+

83+ git clone https://github.com/ambiata/mafia

84+ cd mafia

85+

86+ git reset --hard $MAFIA_VERSION

87+

88+ bin/bootstrap ) || exit $?

89+

90+ MAFIA_PATH_TEMP=$(mktemp --tmpdir=$MAFIA_BIN $MAFIA_FILE-XXXXXX 2>/dev/null || TMPDIR=$MAFIA_BIN mktemp -t $MAFIA_FILE)

91+

92+ clean_up_temp () {

93+ clean_up

94+ rm -f "$MAFIA_PATH_TEMP"

95+ }

96+ trap clean_up_temp EXIT

97+

98+ cp "$MAFIA_TEMP/mafia/.cabal-sandbox/bin/mafia" "$MAFIA_PATH_TEMP"

99+ chmod 755 "$MAFIA_PATH_TEMP"

100+ mv "$MAFIA_PATH_TEMP" "$MAFIA_PATH"

101+

102+ clean_up_temp

103+ }

104+

105+ exec $MAFIA_PATH "$@"

106+ fi

107+}

108+

109+#

110+# The actual start of the script.....

111+#

112+

113+if [ $# -gt 0 ]; then

114+ MODE="$1"

115+else

116+ MODE=""

117+fi

118+

119+case "$MODE" in

120+upgrade) shift; run_upgrade "$@" ;;

121+*) exec_mafia "$@"

122+esac

123+# Version: 360716306a06db842ec022a7b9f161d2208483f0

src/Grenade/Core/Layer.hs+3−8

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE GADTs #-}

43{-# LANGUAGE TypeOperators #-}

@@ -20,7 +19,7 @@ neural network.

2019There are two classes of interest: `UpdateLayer` and `Layer`.

2120

2221`UpdateLayer` is required for all types which are used as a layer

23−in a network. Having no shape information, this class is agnostic

22+in a network. Having no shape information, this class is agnotostic

2423to the input and output data of the layer.

2524

2625An instance of `Layer` on the other hand is required for usage in

@@ -42,10 +41,6 @@ import Control.Monad.Random ( MonadRandom )

4241

4342import Data.List ( foldl' )

4443

45−#if MIN_VERSION_base(4,9,0)

46−import Data.Kind (Type)

47−#endif

48−

4944import Grenade.Core.Shape

5045import Grenade.Core.LearningParameters

5146

@@ -55,7 +50,7 @@ import Grenade.Core.LearningParameters

5550class UpdateLayer x where

5651 -- | The type for the gradient for this layer.

5752 -- Unit if there isn't a gradient to pass back.

58− type Gradient x :: Type

53+ type Gradient x :: *

5954

6055 -- | Update a layer with its gradient and learning parameters

6156 runUpdate :: LearningParameters -> x -> Gradient x -> x

@@ -77,7 +72,7 @@ class UpdateLayer x => Layer x (i :: Shape) (o :: Shape) where

7772 -- | The Wengert tape for this layer. Includes all that is required

7873 -- to generate the back propagated gradients efficiently. As a

7974 -- default, `S i` is fine.

80− type Tape x i o :: Type

75+ type Tape x i o :: *

8176

8277 -- | Used in training and scoring. Take the input from the previous

8378 -- layer, and give the output from this layer.

src/Grenade/Core/Network.hs+5−10

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE BangPatterns #-}

43{-# LANGUAGE GADTs #-}

@@ -34,16 +33,12 @@ module Grenade.Core.Network (

3433import Control.Monad.Random ( MonadRandom )

3534

3635import Data.Singletons

36+import Data.Singletons.Prelude

3737import Data.Serialize

3838

39−#if MIN_VERSION_base(4,9,0)

40−import Data.Kind (Type)

41−#endif

42−

4339import Grenade.Core.Layer

4440import Grenade.Core.LearningParameters

4541import Grenade.Core.Shape

46−import Prelude.Singletons

4742

4843-- | Type of a network.

4944--

@@ -53,7 +48,7 @@ import Prelude.Singletons

5348--

5449-- Can be considered to be a heterogeneous list of layers which are able to

5550-- transform the data shapes of the network.

56−data Network :: [Type] -> [Shape] -> Type where

51+data Network :: [*] -> [Shape] -> * where

5752 NNil :: SingI i

5853 => Network '[] '[i]

5954

@@ -71,7 +66,7 @@ instance (Show x, Show (Network xs rs)) => Show (Network (x ': xs) (i ': rs)) wh

7166-- | Gradient of a network.

7267--

7368-- Parameterised on the layers of the network.

74−data Gradients :: [Type] -> Type where

69+data Gradients :: [*] -> * where

7570 GNil :: Gradients '[]

7671

7772 (:/>) :: UpdateLayer x

@@ -82,7 +77,7 @@ data Gradients :: [Type] -> Type where

8277-- | Wegnert Tape of a network.

8378--

8479-- Parameterised on the layers and shapes of the network.

85−data Tapes :: [Type] -> [Shape] -> Type where

80+data Tapes :: [*] -> [Shape] -> * where

8681 TNil :: SingI i

8782 => Tapes '[] '[i]

8883

@@ -157,7 +152,7 @@ applyUpdate _ NNil GNil

157152

158153-- | A network can easily be created by hand with (:~>), but an easy way to

159154-- initialise a random network is with the randomNetwork.

160−class CreatableNetwork (xs :: [Type]) (ss :: [Shape]) where

155+class CreatableNetwork (xs :: [*]) (ss :: [Shape]) where

161156 -- | Create a network with randomly initialised weights.

162157 --

163158 -- Calls to this function will not compile if the type of the neural

src/Grenade/Core/Runner.hs+1−2

@@ -13,12 +13,11 @@ module Grenade.Core.Runner (

1313 , runNet

1414 ) where

1515

16+import Data.Singletons.Prelude

1617

1718import Grenade.Core.LearningParameters

1819import Grenade.Core.Network

1920import Grenade.Core.Shape

20−import Data.Singletons

21−import Prelude.Singletons

2221

2322-- | Perform reverse automatic differentiation on the network

2423-- for the current input and expected output.

src/Grenade/Core/Shape.hs+21−29

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE GADTs #-}

43{-# LANGUAGE KindSignatures #-}

@@ -8,7 +7,6 @@

87{-# LANGUAGE FlexibleContexts #-}

98{-# LANGUAGE ScopedTypeVariables #-}

109{-# LANGUAGE RankNTypes #-}

11−{-# LANGUAGE UndecidableInstances #-}

1210{-|

1311Module : Grenade.Core.Shape

1412Description : Dependently typed shapes of data which are passed between layers of a network

@@ -19,22 +17,26 @@ Stability : experimental

1917

2018-}

2119module Grenade.Core.Shape (

22− S (..)

23− , Shape (..)

24− , SShape (..)

20+ Shape (..)

21+ , S (..)

22+ , Sing (..)

23+

2524 , randomOfShape

2625 , fromStorable

2726 ) where

2827

2928import Control.DeepSeq (NFData (..))

3029import Control.Monad.Random ( MonadRandom, getRandom )

31−import Data.Kind (Type)

30+

3231import Data.Proxy

3332import Data.Serialize

3433import Data.Singletons

34+import Data.Singletons.TypeLits

3535import Data.Vector.Storable ( Vector )

3636import qualified Data.Vector.Storable as V

37+

3738import GHC.TypeLits

39+

3840import qualified Numeric.LinearAlgebra.Static as H

3941import Numeric.LinearAlgebra.Static

4042import qualified Numeric.LinearAlgebra as NLA

@@ -79,27 +81,17 @@ deriving instance Show (S n)

7981-- These could probably be derived with template haskell, but this seems

8082-- clear and makes adding the KnownNat constraints simple.

8183-- We can also keep our code TH free, which is great.

82−#if MIN_VERSION_singletons(2,6,0)

83−-- In singletons 2.6 Sing switched from a data family to a type family.

84−type instance Sing = SShape

85−

86−data SShape :: Shape -> Type where

87− D1Sing :: KnownNat a => SShape ('D1 a)

88− D2Sing :: (KnownNat a, KnownNat b) => SShape ('D2 a b)

89− D3Sing :: (KnownNat (a * c), KnownNat a, KnownNat b, KnownNat c) => SShape ('D3 a b c)

90−#else

9184data instance Sing (n :: Shape) where

9285 D1Sing :: Sing a -> Sing ('D1 a)

9386 D2Sing :: Sing a -> Sing b -> Sing ('D2 a b)

9487 D3Sing :: KnownNat (a * c) => Sing a -> Sing b -> Sing c -> Sing ('D3 a b c)

95−#endif

9688

9789instance KnownNat a => SingI ('D1 a) where

98− sing = D1Sing

90+ sing = D1Sing sing

9991instance (KnownNat a, KnownNat b) => SingI ('D2 a b) where

100− sing = D2Sing

92+ sing = D2Sing sing sing

10193instance (KnownNat a, KnownNat b, KnownNat c, KnownNat (a * c)) => SingI ('D3 a b c) where

102− sing = D3Sing

94+ sing = D3Sing sing sing sing

10395

10496instance SingI x => Num (S x) where

10597 (+) = n2 (+)

@@ -148,13 +140,13 @@ randomOfShape :: forall x m. ( MonadRandom m, SingI x ) => m (S x)

148140randomOfShape = do

149141 seed :: Int <- getRandom

150142 return $ case (sing :: Sing x) of

151− D1Sing ->

143+ D1Sing SNat ->

152144 S1D (randomVector seed Uniform * 2 - 1)

153145

154− D2Sing ->

146+ D2Sing SNat SNat ->

155147 S2D (uniformSample seed (-1) 1)

156148

157− D3Sing ->

149+ D3Sing SNat SNat SNat ->

158150 S3D (uniformSample seed (-1) 1)

159151

160152-- | Generate a shape from a Storable Vector.

@@ -162,13 +154,13 @@ randomOfShape = do

162154-- Returns Nothing if the vector is of the wrong size.

163155fromStorable :: forall x. SingI x => Vector Double -> Maybe (S x)

164156fromStorable xs = case sing :: Sing x of

165− D1Sing ->

157+ D1Sing SNat ->

166158 S1D <$> H.create xs

167159

168− D2Sing ->

160+ D2Sing SNat SNat ->

169161 S2D <$> mkL xs

170162

171− D3Sing ->

163+ D3Sing SNat SNat SNat ->

172164 S3D <$> mkL xs

173165 where

174166 mkL :: forall rows columns. (KnownNat rows, KnownNat columns)

@@ -205,13 +197,13 @@ n2 f (S2D x) (S2D y) = S2D (f x y)

205197n2 f (S3D x) (S3D y) = S3D (f x y)

206198

207199-- Helper function for creating the number instances

208−nk :: forall x. (SingI x) => Double -> S x

200+nk :: forall x. SingI x => Double -> S x

209201nk x = case (sing :: Sing x) of

210− D1Sing ->

202+ D1Sing SNat ->

211203 S1D (konst x)

212204

213− D2Sing ->

205+ D2Sing SNat SNat ->

214206 S2D (konst x)

215207

216− D3Sing ->

208+ D3Sing SNat SNat SNat ->

217209 S3D (konst x)

src/Grenade/Layers/Concat.hs+1−7

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE GADTs #-}

43{-# LANGUAGE TypeOperators #-}

@@ -9,7 +8,6 @@

98{-# LANGUAGE FlexibleInstances #-}

109{-# LANGUAGE ScopedTypeVariables #-}

1110{-# LANGUAGE StandaloneDeriving #-}

12−{-# LANGUAGE UndecidableInstances #-}

1311{-|

1412Module : Grenade.Layers.Concat

1513Description : Concatenation layer

@@ -28,10 +26,6 @@ import Data.Serialize

2826import Data.Singletons

2927import GHC.TypeLits

3028

31−#if MIN_VERSION_base(4,9,0)

32−import Data.Kind (Type)

33−#endif

34−

3529import Grenade.Core

3630

3731import Numeric.LinearAlgebra.Static ( row, (===), splitRows, unrow, (#), split, R )

@@ -49,7 +43,7 @@ import Numeric.LinearAlgebra.Static ( row, (===), splitRows, unrow, (#

4943--

5044-- 3D images become 3D images with more channels. The sizes must be the same, one can use Pad

5145-- and Crop layers to ensure this is the case.

52−data Concat :: Shape -> Type -> Shape -> Type -> Type where

46+data Concat :: Shape -> * -> Shape -> * -> * where

5347 Concat :: x -> y -> Concat m x n y

5448

5549instance (Show x, Show y) => Show (Concat m x n y) where

src/Grenade/Layers/Convolution.hs+6−7

@@ -1,13 +1,12 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE ScopedTypeVariables #-}

3+{-# LANGUAGE RecordWildCards #-}

44{-# LANGUAGE GADTs #-}

55{-# LANGUAGE TypeOperators #-}

66{-# LANGUAGE TypeFamilies #-}

77{-# LANGUAGE MultiParamTypeClasses #-}

88{-# LANGUAGE FlexibleInstances #-}

99{-# LANGUAGE FlexibleContexts #-}

10−{-# LANGUAGE UndecidableInstances #-}

1110{-|

1211Module : Grenade.Layers.Convolution

1312Description : Convolution layer

@@ -28,9 +27,9 @@ import Control.Monad.Random hiding ( fromList )

2827import Data.Maybe

2928import Data.Proxy

3029import Data.Serialize

30+import Data.Singletons.TypeLits

3131

3232import GHC.TypeLits

33−import Data.Kind (Type)

3433

3534import Numeric.LinearAlgebra hiding ( uniformSample, konst )

3635import qualified Numeric.LinearAlgebra as LA

@@ -57,7 +56,7 @@ data Convolution :: Nat -- Number of channels, for the first layer this could be

5756 -> Nat -- The number of column in the kernel filter

5857 -> Nat -- The row stride of the convolution filter

5958 -> Nat -- The columns stride of the convolution filter

60− -> Type where

59+ -> * where

6160 Convolution :: ( KnownNat channels

6261 , KnownNat filters

6362 , KnownNat kernelRows

@@ -76,7 +75,7 @@ data Convolution' :: Nat -- Number of channels, for the first layer this could b

7675 -> Nat -- The number of column in the kernel filter

7776 -> Nat -- The row stride of the convolution filter

7877 -> Nat -- The columns stride of the convolution filter

79− -> Type where

78+ -> * where

8079 Convolution' :: ( KnownNat channels

8180 , KnownNat filters

8281 , KnownNat kernelRows

@@ -131,8 +130,8 @@ instance ( KnownNat channels

131130 , KnownNat (kernelRows * kernelColumns * channels)

132131 ) => UpdateLayer (Convolution channels filters kernelRows kernelColumns strideRows strideColumns) where

133132 type Gradient (Convolution channels filters kernelRows kernelColumns strideRows strideColumns) = (Convolution' channels filters kernelRows kernelColumns strideRows strideColumns)

134− runUpdate lp (Convolution oldKernel oldMomentum) (Convolution' kernelGradient) =

135− let (newKernel, newMomentum) = descendMatrix (learningRate lp) (learningMomentum lp) (learningRegulariser lp) oldKernel kernelGradient oldMomentum

133+ runUpdate LearningParameters {..} (Convolution oldKernel oldMomentum) (Convolution' kernelGradient) =

134+ let (newKernel, newMomentum) = descendMatrix learningRate learningMomentum learningRegulariser oldKernel kernelGradient oldMomentum

136135 in Convolution newKernel newMomentum

137136

138137 createRandom = randomConvolution

src/Grenade/Layers/Crop.hs+2−10

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE ScopedTypeVariables #-}

43{-# LANGUAGE GADTs #-}

@@ -6,7 +5,6 @@

65{-# LANGUAGE TypeFamilies #-}

76{-# LANGUAGE FlexibleContexts #-}

87{-# LANGUAGE MultiParamTypeClasses #-}

9−{-# LANGUAGE UndecidableInstances #-}

108{-|

119Module : Grenade.Layers.Crop

1210Description : Cropping layer

@@ -20,10 +18,8 @@ module Grenade.Layers.Crop (

2018

2119import Data.Maybe

2220import Data.Proxy

23−import Data.Serialize

24−

21+import Data.Singletons.TypeLits

2522import GHC.TypeLits

26−import Data.Kind (Type)

2723

2824import Grenade.Core

2925import Grenade.Layers.Internal.Pad

@@ -35,7 +31,7 @@ import Numeric.LinearAlgebra.Static (extract, create)

3531data Crop :: Nat

3632 -> Nat

3733 -> Nat

38− -> Nat -> Type where

34+ -> Nat -> * where

3935 Crop :: Crop cropLeft cropTop cropRight cropBottom

4036

4137instance Show (Crop cropLeft cropTop cropRight cropBottom) where

@@ -46,10 +42,6 @@ instance UpdateLayer (Crop l t r b) where

4642 runUpdate _ x _ = x

4743 createRandom = return Crop

4844

49−instance Serialize (Crop l t r b) where

50− put _ = return ()

51− get = return Crop

52−

5345-- | A two dimentional image can be cropped.

5446instance ( KnownNat cropLeft

5547 , KnownNat cropTop

src/Grenade/Layers/Deconvolution.hs+6−7

@@ -1,13 +1,12 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE ScopedTypeVariables #-}

3+{-# LANGUAGE RecordWildCards #-}

44{-# LANGUAGE GADTs #-}

55{-# LANGUAGE TypeOperators #-}

66{-# LANGUAGE TypeFamilies #-}

77{-# LANGUAGE MultiParamTypeClasses #-}

88{-# LANGUAGE FlexibleInstances #-}

99{-# LANGUAGE FlexibleContexts #-}

10−{-# LANGUAGE UndecidableInstances #-}

1110{-|

1211Module : Grenade.Layers.Deconvolution

1312Description : Deconvolution layer

@@ -32,9 +31,9 @@ import Control.Monad.Random hiding ( fromList )

3231import Data.Maybe

3332import Data.Proxy

3433import Data.Serialize

34+import Data.Singletons.TypeLits

3535

3636import GHC.TypeLits

37−import Data.Kind (Type)

3837

3938import Numeric.LinearAlgebra hiding ( uniformSample, konst )

4039import qualified Numeric.LinearAlgebra as LA

@@ -56,7 +55,7 @@ data Deconvolution :: Nat -- Number of channels, for the first layer this could

5655 -> Nat -- The number of column in the kernel filter

5756 -> Nat -- The row stride of the Deconvolution filter

5857 -> Nat -- The columns stride of the Deconvolution filter

59− -> Type where

58+ -> * where

6059 Deconvolution :: ( KnownNat channels

6160 , KnownNat filters

6261 , KnownNat kernelRows

@@ -75,7 +74,7 @@ data Deconvolution' :: Nat -- Number of channels, for the first layer this could

7574 -> Nat -- The number of column in the kernel filter

7675 -> Nat -- The row stride of the Deconvolution filter

7776 -> Nat -- The columns stride of the Deconvolution filter

78− -> Type where

77+ -> * where

7978 Deconvolution' :: ( KnownNat channels

8079 , KnownNat filters

8180 , KnownNat kernelRows

@@ -130,8 +129,8 @@ instance ( KnownNat channels

130129 , KnownNat (kernelRows * kernelColumns * filters)

131130 ) => UpdateLayer (Deconvolution channels filters kernelRows kernelColumns strideRows strideColumns) where

132131 type Gradient (Deconvolution channels filters kernelRows kernelColumns strideRows strideColumns) = (Deconvolution' channels filters kernelRows kernelColumns strideRows strideColumns)

133− runUpdate lp (Deconvolution oldKernel oldMomentum) (Deconvolution' kernelGradient) =

134− let (newKernel, newMomentum) = descendMatrix (learningRate lp) (learningMomentum lp) (learningRegulariser lp) oldKernel kernelGradient oldMomentum

132+ runUpdate LearningParameters {..} (Deconvolution oldKernel oldMomentum) (Deconvolution' kernelGradient) =

133+ let (newKernel, newMomentum) = descendMatrix learningRate learningMomentum learningRegulariser oldKernel kernelGradient oldMomentum

135134 in Deconvolution newKernel newMomentum

136135

137136 createRandom = randomDeconvolution

src/Grenade/Layers/FullyConnected.hs+5−4

@@ -1,4 +1,5 @@

11{-# LANGUAGE DataKinds #-}

2+{-# LANGUAGE RecordWildCards #-}

23{-# LANGUAGE TypeOperators #-}

34{-# LANGUAGE TypeFamilies #-}

45{-# LANGUAGE MultiParamTypeClasses #-}

@@ -13,6 +14,7 @@ import Control.Monad.Random hiding (fromList)

1314

1415import Data.Proxy

1516import Data.Serialize

17+import Data.Singletons.TypeLits

1618

1719import qualified Numeric.LinearAlgebra as LA

1820import Numeric.LinearAlgebra.Static

@@ -20,7 +22,6 @@ import Numeric.LinearAlgebra.Static

2022import Grenade.Core

2123

2224import Grenade.Layers.Internal.Update

23−import GHC.TypeLits

2425

2526-- | A basic fully connected (or inner product) neural network layer.

2627data FullyConnected i o = FullyConnected

@@ -37,9 +38,9 @@ instance Show (FullyConnected i o) where

3738instance (KnownNat i, KnownNat o) => UpdateLayer (FullyConnected i o) where

3839 type Gradient (FullyConnected i o) = (FullyConnected' i o)

3940

40− runUpdate lp (FullyConnected (FullyConnected' oldBias oldActivations) (FullyConnected' oldBiasMomentum oldMomentum)) (FullyConnected' biasGradient activationGradient) =

41− let (newBias, newBiasMomentum) = descendVector (learningRate lp) (learningMomentum lp) (learningRegulariser lp) oldBias biasGradient oldBiasMomentum

42− (newActivations, newMomentum) = descendMatrix (learningRate lp) (learningMomentum lp) (learningRegulariser lp) oldActivations activationGradient oldMomentum

41+ runUpdate LearningParameters {..} (FullyConnected (FullyConnected' oldBias oldActivations) (FullyConnected' oldBiasMomentum oldMomentum)) (FullyConnected' biasGradient activationGradient) =

42+ let (newBias, newBiasMomentum) = descendVector learningRate learningMomentum learningRegulariser oldBias biasGradient oldBiasMomentum

43+ (newActivations, newMomentum) = descendMatrix learningRate learningMomentum learningRegulariser oldActivations activationGradient oldMomentum

4344 in FullyConnected (FullyConnected' newBias newActivations) (FullyConnected' newBiasMomentum newMomentum)

4445

4546 createRandom = randomFullyConnected

src/Grenade/Layers/Merge.hs+1−6

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE GADTs #-}

43{-# LANGUAGE TypeOperators #-}

@@ -24,17 +23,13 @@ import Data.Serialize

2423

2524import Data.Singletons

2625

27−#if MIN_VERSION_base(4,9,0)

28−import Data.Kind (Type)

29−#endif

30−

3126import Grenade.Core

3227

3328-- | A Merging layer.

3429--

3530-- Similar to Concat layer, except sums the activations instead of creating a larger

3631-- shape.

37−data Merge :: Type -> Type -> Type where

32+data Merge :: * -> * -> * where

3833 Merge :: x -> y -> Merge x y

3934

4035instance (Show x, Show y) => Show (Merge x y) where

src/Grenade/Layers/Pad.hs+4−7

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE ScopedTypeVariables #-}

43{-# LANGUAGE GADTs #-}

@@ -6,7 +5,6 @@

65{-# LANGUAGE TypeFamilies #-}

76{-# LANGUAGE MultiParamTypeClasses #-}

87{-# LANGUAGE FlexibleContexts #-}

9−{-# LANGUAGE UndecidableInstances #-}

108{-|

119Module : Grenade.Core.Pad

1210Description : Padding layer for 2D and 3D images

@@ -21,9 +19,8 @@ module Grenade.Layers.Pad (

2119import Data.Maybe

2220import Data.Proxy

2321import Data.Serialize

24−

22+import Data.Singletons.TypeLits

2523import GHC.TypeLits

26−import Data.Kind (Type)

2724

2825import Grenade.Core

2926import Grenade.Layers.Internal.Pad

@@ -37,7 +34,7 @@ import Numeric.LinearAlgebra.Static (extract, create)

3734data Pad :: Nat

3835 -> Nat

3936 -> Nat

40− -> Nat -> Type where

37+ -> Nat -> * where

4138 Pad :: Pad padLeft padTop padRight padBottom

4239

4340instance Show (Pad padLeft padTop padRight padBottom) where

@@ -52,7 +49,7 @@ instance Serialize (Pad l t r b) where

5249 put _ = return ()

5350 get = return Pad

5451

55−-- | A two dimentional image can be padded.

52+-- | A two dimentional image can be padped.

5653instance ( KnownNat padLeft

5754 , KnownNat padTop

5855 , KnownNat padRight

@@ -82,7 +79,7 @@ instance ( KnownNat padLeft

8279 vs = subMatrix (padt, padl) (nrows, ncols) m

8380 in ((), S2D . fromJust . create $ vs)

8481

85−-- | A two dimentional image can be padded.

82+-- | A two dimentional image can be padped.

8683instance ( KnownNat padLeft

8784 , KnownNat padTop

8885 , KnownNat padRight

src/Grenade/Layers/Pooling.hs+2−5

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE ScopedTypeVariables #-}

43{-# LANGUAGE StandaloneDeriving #-}

@@ -7,7 +6,6 @@

76{-# LANGUAGE TypeFamilies #-}

87{-# LANGUAGE MultiParamTypeClasses #-}

98{-# LANGUAGE FlexibleContexts #-}

10−{-# LANGUAGE UndecidableInstances #-}

119{-|

1210Module : Grenade.Core.Pooling

1311Description : Max Pooling layer for 2D and 3D images

@@ -22,9 +20,8 @@ module Grenade.Layers.Pooling (

2220import Data.Maybe

2321import Data.Proxy

2422import Data.Serialize

25−

23+import Data.Singletons.TypeLits

2624import GHC.TypeLits

27−import Data.Kind (Type)

2825

2926import Grenade.Core

3027import Grenade.Layers.Internal.Pooling

@@ -39,7 +36,7 @@ import Numeric.LinearAlgebra.Static as LAS hiding ((|||), build, toRow

3936-- The kernel size dictates which input and output sizes will "fit". Fitting the equation:

4037-- `out = (in - kernel) / stride + 1` for both dimensions.

4138--

42−data Pooling :: Nat -> Nat -> Nat -> Nat -> Type where

39+data Pooling :: Nat -> Nat -> Nat -> Nat -> * where

4340 Pooling :: Pooling kernelRows kernelColumns strideRows strideColumns

4441

4542instance Show (Pooling k k' s s') where

src/Grenade/Layers/Reshape.hs+1−1

@@ -3,7 +3,6 @@

33{-# LANGUAGE TypeFamilies #-}

44{-# LANGUAGE MultiParamTypeClasses #-}

55{-# LANGUAGE FlexibleContexts #-}

6−{-# LANGUAGE UndecidableInstances #-}

76{-|

87Module : Grenade.Layers.Reshape

98Description : Multipurpose reshaping layer

@@ -17,6 +16,7 @@ module Grenade.Layers.Reshape (

1716

1817import Data.Serialize

1918

19+import Data.Singletons.TypeLits

2020import GHC.TypeLits

2121

2222import Numeric.LinearAlgebra.Static

src/Grenade/Layers/Sinusoid.hsdeleted−39

@@ -1,39 +0,0 @@

1−{-# LANGUAGE DataKinds #-}

2−{-# LANGUAGE FlexibleInstances #-}

3−{-# LANGUAGE MultiParamTypeClasses #-}

4−{-# LANGUAGE TypeFamilies #-}

5−{-# LANGUAGE TypeOperators #-}

6−{-|

7−Module : Grenade.Layers.Sinusoid

8−Description : Sinusoid nonlinear layer

9−Copyright : (c) Manuel Schneckenreither, 2018

10−License : BSD2

11−Stability : experimental

12−-}

13−module Grenade.Layers.Sinusoid (

14− Sinusoid (..)

15− ) where

16−

17−import Data.Serialize

18−import Data.Singletons

19−

20−import Grenade.Core

21−

22−-- | A Sinusoid layer.

23−-- A layer which can act between any shape of the same dimension, performing a sin function.

24−data Sinusoid = Sinusoid

25− deriving Show

26−

27−instance UpdateLayer Sinusoid where

28− type Gradient Sinusoid = ()

29− runUpdate _ _ _ = Sinusoid

30− createRandom = return Sinusoid

31−

32−instance Serialize Sinusoid where

33− put _ = return ()

34− get = return Sinusoid

35−

36−instance (a ~ b, SingI a) => Layer Sinusoid a b where

37− type Tape Sinusoid a b = S a

38− runForwards _ a = (a, sin a)

39− runBackwards _ a g = ((), cos a * g)

src/Grenade/Layers/Tanh.hs+3−3

@@ -1,8 +1,8 @@

11{-# LANGUAGE DataKinds #-}

2+{-# LANGUAGE TypeOperators #-}

3+{-# LANGUAGE TypeFamilies #-}

24{-# LANGUAGE FlexibleInstances #-}

35{-# LANGUAGE MultiParamTypeClasses #-}

4−{-# LANGUAGE TypeFamilies #-}

5−{-# LANGUAGE TypeOperators #-}

66{-|

77Module : Grenade.Layers.Tanh

88Description : Hyperbolic tangent nonlinear layer

@@ -20,7 +20,7 @@ import Data.Singletons

2020import Grenade.Core

2121

2222-- | A Tanh layer.

23−-- A layer which can act between any shape of the same dimension, performing a tanh function.

23+-- A layer which can act between any shape of the same dimension, perfoming a tanh function.

2424data Tanh = Tanh

2525 deriving Show

2626

src/Grenade/Recurrent/Core/Layer.hs+2−7

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE TypeFamilies #-}

43{-# LANGUAGE MultiParamTypeClasses #-}

@@ -9,10 +8,6 @@ module Grenade.Recurrent.Core.Layer (

98 , RecurrentUpdateLayer (..)

109 ) where

1110

12−#if MIN_VERSION_base(4,9,0)

13−import Data.Kind (Type)

14−#endif

15−

1611import Grenade.Core

1712

1813-- | Class for a recurrent layer.

@@ -20,11 +15,11 @@ import Grenade.Core

2015-- of an extra recurrent data shape.

2116class UpdateLayer x => RecurrentUpdateLayer x where

2217 -- | Shape of data that is passed between each subsequent run of the layer

23− type RecurrentShape x :: Type

18+ type RecurrentShape x :: *

2419

2520class (RecurrentUpdateLayer x, Num (RecurrentShape x)) => RecurrentLayer x (i :: Shape) (o :: Shape) where

2621 -- | Wengert Tape

27− type RecTape x i o :: Type

22+ type RecTape x i o :: *

2823 -- | Used in training and scoring. Take the input from the previous

2924 -- layer, and give the output from this layer.

3025 runRecurrentForwards :: x -> RecurrentShape x -> S i -> (RecTape x i o, RecurrentShape x, S o)

src/Grenade/Recurrent/Core/Network.hs+10−12

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE GADTs #-}

43{-# LANGUAGE TypeOperators #-}

@@ -29,23 +28,22 @@ module Grenade.Recurrent.Core.Network (

2928

3029

3130import Control.Monad.Random ( MonadRandom )

31+import Data.Singletons ( SingI )

32+import Data.Singletons.Prelude ( Head, Last )

3233import Data.Serialize

3334

34−import Data.Kind (Type)

35−

3635import Grenade.Core

3736import Grenade.Recurrent.Core.Layer

38−import Prelude.Singletons

3937

4038-- | Witness type to say indicate we're building up with a normal feed

4139-- forward layer.

42−data FeedForward :: Type -> Type

40+data FeedForward :: * -> *

4341-- | Witness type to say indicate we're building up with a recurrent layer.

44−data Recurrent :: Type -> Type

42+data Recurrent :: * -> *

4543

4644-- | Type of a recurrent neural network.

4745--

48−-- The [Type] type specifies the types of the layers.

46+-- The [*] type specifies the types of the layers.

4947--

5048-- The [Shape] type specifies the shapes of data passed between the layers.

5149--

@@ -54,7 +52,7 @@ data Recurrent :: Type -> Type

5452--

5553-- Often, to make the definitions more concise, one will use a type alias

5654-- for these empty data types.

57−data RecurrentNetwork :: [Type] -> [Shape] -> Type where

55+data RecurrentNetwork :: [*] -> [Shape] -> * where

5856 RNil :: SingI i

5957 => RecurrentNetwork '[] '[i]

6058

@@ -73,7 +71,7 @@ infixr 5 :~@>

7371-- | Gradient of a network.

7472--

7573-- Parameterised on the layers of the network.

76−data RecurrentGradient :: [Type] -> Type where

74+data RecurrentGradient :: [*] -> * where

7775 RGNil :: RecurrentGradient '[]

7876

7977 (://>) :: UpdateLayer x

@@ -83,7 +81,7 @@ data RecurrentGradient :: [Type] -> Type where

8381

8482-- | Recurrent inputs (sideways shapes on an imaginary unrolled graph)

8583-- Parameterised on the layers of a Network.

86−data RecurrentInputs :: [Type] -> Type where

84+data RecurrentInputs :: [*] -> * where

8785 RINil :: RecurrentInputs '[]

8886

8987 (:~~+>) :: (UpdateLayer x, Fractional (RecurrentInputs xs))

@@ -97,7 +95,7 @@ data RecurrentInputs :: [Type] -> Type where

9795--

9896-- We index on the time step length as well, to ensure

9997-- that that all Tape lengths are the same.

100−data RecurrentTape :: [Type] -> [Shape] -> Type where

98+data RecurrentTape :: [*] -> [Shape] -> * where

10199 TRNil :: SingI i

102100 => RecurrentTape '[] '[i]

103101

@@ -206,7 +204,7 @@ instance (Show x, Show (RecurrentNetwork xs rs)) => Show (RecurrentNetwork (Recu

206204

207205-- | A network can easily be created by hand with (:~~>) and (:~@>), but an easy way to initialise a random

208206-- recurrent network and a set of random inputs for it is with the randomRecurrent.

209−class CreatableRecurrent (xs :: [Type]) (ss :: [Shape]) where

207+class CreatableRecurrent (xs :: [*]) (ss :: [Shape]) where

210208 -- | Create a network of the types requested

211209 randomRecurrent :: MonadRandom m => m (RecurrentNetwork xs ss)

212210

src/Grenade/Recurrent/Core/Runner.hs+6−5

@@ -6,6 +6,7 @@

66{-# LANGUAGE TypeFamilies #-}

77{-# LANGUAGE FlexibleContexts #-}

88{-# LANGUAGE RankNTypes #-}

9+{-# LANGUAGE RecordWildCards #-}

910

1011module Grenade.Recurrent.Core.Runner (

1112 runRecurrentExamples

@@ -17,10 +18,10 @@ module Grenade.Recurrent.Core.Runner (

1718 ) where

1819

1920import Data.List ( foldl' )

21+import Data.Singletons.Prelude

2022import Grenade.Core

2123

2224import Grenade.Recurrent.Core.Network

23−import Prelude.Singletons

2425

2526type RecurrentGradients layers = [RecurrentGradient layers]

2627

@@ -100,11 +101,11 @@ updateRecInputs :: Fractional (RecurrentInputs sublayers)

100101 -> RecurrentInputs sublayers

101102 -> RecurrentInputs sublayers

102103

103−updateRecInputs lp (() :~~+> xs) (() :~~+> ys)

104− = () :~~+> updateRecInputs lp xs ys

104+updateRecInputs l@LearningParameters {..} (() :~~+> xs) (() :~~+> ys)

105+ = () :~~+> updateRecInputs l xs ys

105106

106−updateRecInputs lp (x :~@+> xs) (y :~@+> ys)

107− = (realToFrac (1 - learningRate lp * learningRegulariser lp) * x - realToFrac (learningRate lp) * y) :~@+> updateRecInputs lp xs ys

107+updateRecInputs l@LearningParameters {..} (x :~@+> xs) (y :~@+> ys)

108+ = (realToFrac (1 - learningRate * learningRegulariser) * x - realToFrac learningRate * y) :~@+> updateRecInputs l xs ys

108109

109110updateRecInputs _ RINil RINil

110111 = RINil

src/Grenade/Recurrent/Layers/BasicRecurrent.hs+8−9

@@ -1,6 +1,6 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE GADTs #-}

3+{-# LANGUAGE RecordWildCards #-}

44{-# LANGUAGE TypeOperators #-}

55{-# LANGUAGE TypeFamilies #-}

66{-# LANGUAGE MultiParamTypeClasses #-}

@@ -16,8 +16,7 @@ module Grenade.Recurrent.Layers.BasicRecurrent (

1616

1717import Control.Monad.Random ( MonadRandom, getRandom )

1818

19−

20−import Data.Kind (Type)

19+import Data.Singletons.TypeLits

2120

2221import Numeric.LinearAlgebra.Static

2322

@@ -28,7 +27,7 @@ import Grenade.Recurrent.Core

2827

2928data BasicRecurrent :: Nat -- Input layer size

3029 -> Nat -- Output layer size

31− -> Type where

30+ -> * where

3231 BasicRecurrent :: ( KnownNat input

3332 , KnownNat output

3433 , KnownNat matrixCols

@@ -41,7 +40,7 @@ data BasicRecurrent :: Nat -- Input layer size

4140

4241data BasicRecurrent' :: Nat -- Input layer size

4342 -> Nat -- Output layer size

44− -> Type where

43+ -> * where

4544 BasicRecurrent' :: ( KnownNat input

4645 , KnownNat output

4746 , KnownNat matrixCols

@@ -56,11 +55,11 @@ instance Show (BasicRecurrent i o) where

5655instance (KnownNat i, KnownNat o, KnownNat (i + o)) => UpdateLayer (BasicRecurrent i o) where

5756 type Gradient (BasicRecurrent i o) = (BasicRecurrent' i o)

5857

59− runUpdate lp (BasicRecurrent oldBias oldBiasMomentum oldActivations oldMomentum) (BasicRecurrent' biasGradient activationGradient) =

60− let newBiasMomentum = konst (learningMomentum lp) * oldBiasMomentum - konst (learningRate lp) * biasGradient

58+ runUpdate LearningParameters {..} (BasicRecurrent oldBias oldBiasMomentum oldActivations oldMomentum) (BasicRecurrent' biasGradient activationGradient) =

59+ let newBiasMomentum = konst learningMomentum * oldBiasMomentum - konst learningRate * biasGradient

6160 newBias = oldBias + newBiasMomentum

62− newMomentum = konst (learningMomentum lp) * oldMomentum - konst (learningRate lp) * activationGradient

63− regulariser = konst (learningRegulariser lp * learningRate lp) * oldActivations

61+ newMomentum = konst learningMomentum * oldMomentum - konst learningRate * activationGradient

62+ regulariser = konst (learningRegulariser * learningRate) * oldActivations

6463 newActivations = oldActivations + newMomentum - regulariser

6564 in BasicRecurrent newBias newBiasMomentum newActivations newMomentum

6665

src/Grenade/Recurrent/Layers/ConcatRecurrent.hs+1−7

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE DataKinds #-}

32{-# LANGUAGE GADTs #-}

43{-# LANGUAGE TypeOperators #-}

@@ -9,7 +8,6 @@

98{-# LANGUAGE FlexibleInstances #-}

109{-# LANGUAGE ScopedTypeVariables #-}

1110{-# LANGUAGE StandaloneDeriving #-}

12−{-# LANGUAGE UndecidableInstances #-}

1311{-|

1412Module : Grenade.Layers.Concat

1513Description : Concatenation layer

@@ -28,10 +26,6 @@ import Data.Serialize

2826import Data.Singletons

2927import GHC.TypeLits

3028

31−#if MIN_VERSION_base(4,9,0)

32−import Data.Kind (Type)

33−#endif

34−

3529import Grenade.Core

3630import Grenade.Recurrent.Core

3731

@@ -50,7 +44,7 @@ import Numeric.LinearAlgebra.Static ( (#), split, R )

5044--

5145-- 3D images become 3D images with more channels. The sizes must be the same, one can use Pad

5246-- and Crop layers to ensure this is the case.

53−data ConcatRecurrent :: Shape -> Type -> Shape -> Type -> Type where

47+data ConcatRecurrent :: Shape -> * -> Shape -> * -> * where

5448 ConcatRecLeft :: x -> y -> ConcatRecurrent m (Recurrent x) n (FeedForward y)

5549 ConcatRecRight :: x -> y -> ConcatRecurrent m (FeedForward x) n (Recurrent y)

5650 ConcatRecBoth :: x -> y -> ConcatRecurrent m (Recurrent x) n (Recurrent y)

src/Grenade/Recurrent/Layers/LSTM.hs+54−45

@@ -1,8 +1,8 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE BangPatterns #-}

32{-# LANGUAGE DataKinds #-}

43{-# LANGUAGE GADTs #-}

54{-# LANGUAGE RankNTypes #-}

5+{-# LANGUAGE RecordWildCards #-}

66{-# LANGUAGE TypeOperators #-}

77{-# LANGUAGE TypeFamilies #-}

88{-# LANGUAGE MultiParamTypeClasses #-}

@@ -21,7 +21,7 @@ import Control.Monad.Random ( MonadRandom, getRandom )

2121-- import Data.List ( foldl1' )

2222import Data.Proxy

2323import Data.Serialize

24−import Data.Kind (Type)

24+import Data.Singletons.TypeLits

2525

2626import qualified Numeric.LinearAlgebra as LA

2727import Numeric.LinearAlgebra.Static

@@ -29,7 +29,6 @@ import Numeric.LinearAlgebra.Static

2929import Grenade.Core

3030import Grenade.Recurrent.Core

3131import Grenade.Layers.Internal.Update

32−import GHC.TypeLits

3332

3433

3534-- | Long Short Term Memory Recurrent unit

@@ -37,14 +36,14 @@ import GHC.TypeLits

3736-- This is a Peephole formulation, so the recurrent shape is

3837-- just the cell state, the previous output is not held or used

3938-- at all.

40−data LSTM :: Nat -> Nat -> Type where

39+data LSTM :: Nat -> Nat -> * where

4140 LSTM :: ( KnownNat input

4241 , KnownNat output

4342 ) => !(LSTMWeights input output) -- Weights

4443 -> !(LSTMWeights input output) -- Momentums

4544 -> LSTM input output

4645

47−data LSTMWeights :: Nat -> Nat -> Type where

46+data LSTMWeights :: Nat -> Nat -> * where

4847 LSTMWeights :: ( KnownNat input

4948 , KnownNat output

5049 ) => {

@@ -71,7 +70,7 @@ instance (KnownNat i, KnownNat o) => UpdateLayer (LSTM i o) where

7170

7271 -- Run the update function for each group matrix/vector of weights, momentums and gradients.

7372 -- Hmm, maybe the function should be used instead of passing in the learning parameters.

74− runUpdate lp (LSTM w m) g =

73+ runUpdate LearningParameters {..} (LSTM w m) g =

7574 let (wf, wf') = u lstmWf w m g

7675 (uf, uf') = u lstmUf w m g

7776 (bf, bf') = v lstmBf w m g

@@ -88,11 +87,11 @@ instance (KnownNat i, KnownNat o) => UpdateLayer (LSTM i o) where

8887 -- Utility function for updating with the momentum, gradients, and weights.

8988 u :: forall x ix out. (KnownNat ix, KnownNat out) => (x -> (L out ix)) -> x -> x -> x -> ((L out ix), (L out ix))

9089 u e (e -> weights) (e -> momentum) (e -> gradient) =

91− descendMatrix (learningRate lp) (learningMomentum lp) (learningRegulariser lp) weights gradient momentum

90+ descendMatrix learningRate learningMomentum learningRegulariser weights gradient momentum

9291

9392 v :: forall x ix. (KnownNat ix) => (x -> (R ix)) -> x -> x -> x -> ((R ix), (R ix))

9493 v e (e -> weights) (e -> momentum) (e -> gradient) =

95− descendVector (learningRate lp) (learningMomentum lp) (learningRegulariser lp) weights gradient momentum

94+ descendVector learningRate learningMomentum learningRegulariser weights gradient momentum

9695

9796 -- There's a lot of updates here, so to try and minimise the number of data copies

9897 -- we'll create a mutable bucket for each.

@@ -133,18 +132,18 @@ instance (KnownNat i, KnownNat o) => RecurrentLayer (LSTM i o) ('D1 i) ('D1 o) w

133132 type RecTape (LSTM i o) ('D1 i) ('D1 o) = (R o, R i, R o, R o, R o, R o, R o, R o, R o, R o, R o)

134133 -- Forward propagation for the LSTM layer.

135134 -- The size of the cell state is also the size of the output.

136− runRecurrentForwards (LSTM lw _) (S1D cell) (S1D input) =

135+ runRecurrentForwards (LSTM (LSTMWeights {..}) _) (S1D cell) (S1D input) =

137136 let -- Forget state vector

138− f_s = lstmBf lw + lstmWf lw #> input + lstmUf lw #> cell

137+ f_s = lstmBf + lstmWf #> input + lstmUf #> cell

139138 f_t = sigmoid f_s

140139 -- Input state vector

141− i_s = lstmBi lw + lstmWi lw #> input + lstmUi lw #> cell

140+ i_s = lstmBi + lstmWi #> input + lstmUi #> cell

142141 i_t = sigmoid i_s

143142 -- Output state vector

144− o_s = lstmBo lw + lstmWo lw #> input + lstmUo lw #> cell

143+ o_s = lstmBo + lstmWo #> input + lstmUo #> cell

145144 o_t = sigmoid o_s

146145 -- Cell input state vector

147− c_s = lstmBc lw + lstmWc lw #> input

146+ c_s = lstmBc + lstmWc #> input

148147 c_x = tanh c_s

149148 -- Cell state

150149 c_t = f_t * cell + i_t * c_x

@@ -158,7 +157,7 @@ instance (KnownNat i, KnownNat o) => RecurrentLayer (LSTM i o) ('D1 i) ('D1 o) w

158157 --

159158 -- There's a test version using the AD library without hmatrix in the test

160159 -- suite. These should match always.

161− runRecurrentBackwards (LSTM lw _) (cell, input, f_s, f_t, i_s, i_t, o_s, o_t, c_s, c_x, c_t) (S1D cellGrad) (S1D h_t') =

160+ runRecurrentBackwards (LSTM (LSTMWeights {..}) _) (cell, input, f_s, f_t, i_s, i_t, o_s, o_t, c_s, c_x, c_t) (S1D cellGrad) (S1D h_t') =

162161 let -- Reverse Mode AD Derivitives

163162 c_t' = h_t' * o_t + cellGrad

164163

@@ -175,8 +174,8 @@ instance (KnownNat i, KnownNat o) => RecurrentLayer (LSTM i o) ('D1 i) ('D1 o) w

175174 c_s' = tanh' c_s * c_x'

176175

177176 -- The derivatives to pass sideways (recurrent) and downwards

178− cell' = tr (lstmUf lw) #> f_s' + tr (lstmUo lw) #> o_s' + tr (lstmUi lw) #> i_s' + c_t' * f_t

179− input' = tr (lstmWf lw) #> f_s' + tr (lstmWo lw) #> o_s' + tr (lstmWi lw) #> i_s' + tr (lstmWc lw) #> c_s'

177+ cell' = tr lstmUf #> f_s' + tr lstmUo #> o_s' + tr lstmUi #> i_s' + c_t' * f_t

178+ input' = tr lstmWf #> f_s' + tr lstmWo #> o_s' + tr lstmWi #> i_s' + tr lstmWc #> c_s'

180179

181180 -- Calculate the gradient Matricies for the input

182181 lstmWf' = f_s' `outer` input

@@ -235,34 +234,44 @@ tanh' :: (Floating a) => a -> a

235234tanh' t = 1 - s ^ (2 :: Int) where s = tanh t

236235

237236instance (KnownNat i, KnownNat o) => Serialize (LSTM i o) where

238− put (LSTM lw _) = do

239− u (lstmWf lw)

240− u (lstmUf lw)

241− v (lstmBf lw)

242− u (lstmWi lw)

243− u (lstmUi lw)

244− v (lstmBi lw)

245− u (lstmWo lw)

246− u (lstmUo lw)

247− v (lstmBo lw)

248− u (lstmWc lw)

249− v (lstmBc lw)

250− where

251− u :: forall a b. (KnownNat a, KnownNat b) => Putter (L b a)

252− u = putListOf put . LA.toList . LA.flatten . extract

253− v :: forall a. (KnownNat a) => Putter (R a)

254− v = putListOf put . LA.toList . extract

237+ put (LSTM LSTMWeights {..} _) = do

238+ u lstmWf

239+ u lstmUf

240+ v lstmBf

241+ u lstmWi

242+ u lstmUi

243+ v lstmBi

244+ u lstmWo

245+ u lstmUo

246+ v lstmBo

247+ u lstmWc

248+ v lstmBc

249+ where

250+ u :: forall a b. (KnownNat a, KnownNat b) => Putter (L b a)

251+ u = putListOf put . LA.toList . LA.flatten . extract

252+ v :: forall a. (KnownNat a) => Putter (R a)

253+ v = putListOf put . LA.toList . extract

255254

256255 get = do

257− w <- LSTMWeights <$> u <*> u <*> v <*> u <*> u <*> v <*> u <*> u <*> v <*> u <*> v

258− return $ LSTM w (LSTMWeights w0 u0 v0 w0 u0 v0 w0 u0 v0 w0 v0)

259− where

260− u :: forall a b. (KnownNat a, KnownNat b) => Get (L b a)

261− u = let f = fromIntegral $ natVal (Proxy :: Proxy a)

262− in maybe (fail "Vector of incorrect size") return . create . LA.reshape f . LA.fromList =<< getListOf get

263− v :: forall a. (KnownNat a) => Get (R a)

264− v = maybe (fail "Vector of incorrect size") return . create . LA.fromList =<< getListOf get

265−

266− w0 = konst 0

267− u0 = konst 0

268− v0 = konst 0

256+ lstmWf <- u

257+ lstmUf <- u

258+ lstmBf <- v

259+ lstmWi <- u

260+ lstmUi <- u

261+ lstmBi <- v

262+ lstmWo <- u

263+ lstmUo <- u

264+ lstmBo <- v

265+ lstmWc <- u

266+ lstmBc <- v

267+ return $ LSTM (LSTMWeights {..}) (LSTMWeights w0 u0 v0 w0 u0 v0 w0 u0 v0 w0 v0)

268+ where

269+ u :: forall a b. (KnownNat a, KnownNat b) => Get (L b a)

270+ u = let f = fromIntegral $ natVal (Proxy :: Proxy a)

271+ in maybe (fail "Vector of incorrect size") return . create . LA.reshape f . LA.fromList =<< getListOf get

272+ v :: forall a. (KnownNat a) => Get (R a)

273+ v = maybe (fail "Vector of incorrect size") return . create . LA.fromList =<< getListOf get

274+

275+ w0 = konst 0

276+ u0 = konst 0

277+ v0 = konst 0

src/Grenade/Utils/OneHot.hs+4−5

@@ -22,6 +22,7 @@ import Data.Map ( Map )

2222import qualified Data.Map as M

2323

2424import Data.Proxy

25+import Data.Singletons.TypeLits

2526

2627import Data.Vector ( Vector )

2728import qualified Data.Vector as V

@@ -30,7 +31,6 @@ import qualified Data.Vector.Storable as VS

3031import Numeric.LinearAlgebra ( maxIndex )

3132import Numeric.LinearAlgebra.Devel

3233import Numeric.LinearAlgebra.Static

33−import GHC.TypeLits

3434

3535import Grenade.Core.Shape

3636

@@ -52,16 +52,15 @@ oneHot hot =

5252

5353-- | Create a one hot map from any enumerable.

5454-- Returns a map, and the ordered list for the reverse transformation

55−hotMap :: (Ord a, KnownNat n) => Proxy n -> [a] -> Either String (Map a Int, Vector a)

55+hotMap :: (Ord a, KnownNat n) => Proxy n -> [a] -> Maybe (Map a Int, Vector a)

5656hotMap n as =

5757 let len = fromIntegral $ natVal n

5858 uniq = [ c | (c:_) <- group $ sort as]

5959 hotl = length uniq

6060 in if hotl == len

6161 then

62− Right (M.fromList $ zip uniq [0..], V.fromList uniq)

63− else

64− Left ("Couldn't create hotMap of size " ++ show len ++ " from vector with " ++ show hotl ++ " unique characters")

62+ Just (M.fromList $ zip uniq [0..], V.fromList uniq)

63+ else Nothing

6564

6665-- | From a map and value, create a 1D Shape

6766-- with one index hot (1) with the rest 0.

stack.yamldeleted−78

@@ -1,78 +0,0 @@

1−# This file was automatically generated by 'stack init'

2−#

3−# Some commonly used options have been documented as comments in this file.

4−# For advanced use and comprehensive documentation of the format, please see:

5−# https://docs.haskellstack.org/en/stable/yaml_configuration/

6−

7−# A warning or info to be displayed to the user on config load.

8−# user-message: |

9−

10−# Resolver to choose a 'specific' stackage snapshot or a compiler version.

11−# A snapshot resolver dictates the compiler version and the set of packages

12−# to be used for project dependencies. For example:

13−#

14−# resolver: lts-3.5

15−# resolver: nightly-2015-09-21

16−# resolver: ghc-7.10.2

17−#

18−# The location of a snapshot can be provided as a file or url. Stack assumes

19−# a snapshot provided as a file might change, whereas a url resource does not.

20−#

21−# resolver: ./custom-snapshot.yaml

22−# resolver: https://example.com/snapshots/2018-01-01.yaml

23−resolver: lts-20.18

24−

25−# User packages to be built.

26−# Various formats can be used as shown in the example below.

27−#

28−# packages:

29−# - some-directory

30−# - https://example.com/foo/bar/baz-0.0.2.tar.gz

31−# subdirs:

32−# - auto-update

33−# - wai

34−packages:

35− - .

36− - examples

37−

38−# Dependency packages to be pulled from upstream that are not in the resolver.

39−# These entries can reference officially published versions as well as

40−# forks / in-progress versions pinned to a git hash. For example:

41−#

42−# extra-deps:

43−# - acme-missiles-0.3

44−# - git: https://github.com/commercialhaskell/stack.git

45−# commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a

46−#

47−extra-deps:

48− - typelits-witnesses-0.3.0.3

49−

50−# Override default flag values for local packages and extra-deps

51−# flags: {}

52−

53−# Extra package databases containing global packages

54−# extra-package-dbs: []

55−

56−# Control whether we use the GHC we find on the path

57−# system-ghc: true

58−#

59−# Require a specific version of stack, using version ranges

60−# require-stack-version: -any # Default

61−# require-stack-version: ">=2.1"

62−#

63−# Override the architecture used by stack, especially useful on Windows

64−# arch: i386

65−# arch: x86_64

66−#

67−# Extra directories used by stack for building

68−# extra-include-dirs: [/path/to/dir]

69−# extra-lib-dirs: [/path/to/dir]

70−#

71−# Allow a newer minor version of GHC than the snapshot specifies

72−# compiler-check: newer-minor

73−

74−nix:

75− enable: true

76− packages:

77− - blas

78− - lapack

stack.yaml.lockdeleted−19

@@ -1,19 +0,0 @@

1−# This file was autogenerated by Stack.

2−# You should not edit this file by hand.

3−# For more information, please see the documentation at:

4−# https://docs.haskellstack.org/en/stable/lock_files

5−

6−packages:

7−- completed:

8− hackage: typelits-witnesses-0.3.0.3@sha256:2d9df4ac6ff3077bfd2bf659e4b495e157723ac5b45c519762853f55df5c16db,2738

9− pantry-tree:

10− sha256: 6a42a462f98e94933b6e9721acd912c6c6b6a4743635efd15cc5871908c816a0

11− size: 469

12− original:

13− hackage: typelits-witnesses-0.3.0.3

14−snapshots:

15−- completed:

16− sha256: 9fa4bece7acfac1fc7930c5d6e24606004b09e80aa0e52e9f68b148201008db9

17− size: 649606

18− url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/20/18.yaml

19− original: lts-20.18

test/Test/Grenade/Layers/Convolution.hs+1−6

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE TemplateHaskell #-}

32{-# LANGUAGE DataKinds #-}

43{-# LANGUAGE GADTs #-}

@@ -16,10 +15,6 @@ import Data.Singletons ()

1615import GHC.TypeLits

1716import GHC.TypeLits.Witnesses

1817

19−#if MIN_VERSION_base(4,9,0)

20−import Data.Kind (Type)

21−#endif

22−

2318import Grenade.Core

2419import Grenade.Layers.Convolution

2520

@@ -30,7 +25,7 @@ import Test.Hedgehog.Hmatrix

3025import Test.Hedgehog.TypeLits

3126import Test.Hedgehog.Compat

3227

33−data OpaqueConvolution :: Type where

28+data OpaqueConvolution :: * where

3429 OpaqueConvolution :: Convolution channels filters kernelRows kernelColumns strideRows strideColumns -> OpaqueConvolution

3530

3631instance Show OpaqueConvolution where

test/Test/Grenade/Layers/FullyConnected.hs+1−6

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE BangPatterns #-}

32{-# LANGUAGE TemplateHaskell #-}

43{-# LANGUAGE DataKinds #-}

@@ -13,10 +12,6 @@ import Data.Singletons ()

1312

1413import GHC.TypeLits

1514

16−#if MIN_VERSION_base(4,9,0)

17−import Data.Kind (Type)

18−#endif

19−

2015import Grenade.Core

2116import Grenade.Layers.FullyConnected

2217

@@ -25,7 +20,7 @@ import Hedgehog

2520import Test.Hedgehog.Compat

2621import Test.Hedgehog.Hmatrix

2722

28−data OpaqueFullyConnected :: Type where

23+data OpaqueFullyConnected :: * where

2924 OpaqueFullyConnected :: (KnownNat i, KnownNat o) => FullyConnected i o -> OpaqueFullyConnected

3025

3126instance Show OpaqueFullyConnected where

test/Test/Grenade/Layers/PadCrop.hs+4

@@ -7,6 +7,10 @@

77{-# LANGUAGE ScopedTypeVariables #-}

88{-# OPTIONS_GHC -fno-warn-missing-signatures #-}

99

10+#if __GLASGOW_HASKELL__ < 800

11+{-# OPTIONS_GHC -fno-warn-incomplete-patterns #-}

12+#endif

13+

1014module Test.Grenade.Layers.PadCrop where

1115

1216import Grenade

test/Test/Grenade/Layers/Pooling.hs+5−10

@@ -1,4 +1,3 @@

1−{-# LANGUAGE CPP #-}

21{-# LANGUAGE TemplateHaskell #-}

32{-# LANGUAGE DataKinds #-}

43{-# LANGUAGE KindSignatures #-}

@@ -10,10 +9,6 @@ module Test.Grenade.Layers.Pooling where

109import Data.Proxy

1110import Data.Singletons ()

1211

13−#if MIN_VERSION_base(4,9,0)

14−import Data.Kind (Type)

15−#endif

16−

1712import GHC.TypeLits

1813import Grenade.Layers.Pooling

1914

@@ -21,7 +16,7 @@ import Hedgehog

2116

2217import Test.Hedgehog.Compat

2318

24−data OpaquePooling :: Type where

19+data OpaquePooling :: * where

2520 OpaquePooling :: (KnownNat kh, KnownNat kw, KnownNat sh, KnownNat sw) => Pooling kh kw sh sw -> OpaquePooling

2621

2722instance Show OpaquePooling where

@@ -29,10 +24,10 @@ instance Show OpaquePooling where

2924

3025genOpaquePooling :: Gen OpaquePooling

3126genOpaquePooling = do

32− ~(Just kernelHeight) <- someNatVal <$> choose 2 15

33− ~(Just kernelWidth ) <- someNatVal <$> choose 2 15

34− ~(Just strideHeight) <- someNatVal <$> choose 2 15

35− ~(Just strideWidth ) <- someNatVal <$> choose 2 15

27+ Just kernelHeight <- someNatVal <$> choose 2 15

28+ Just kernelWidth <- someNatVal <$> choose 2 15

29+ Just strideHeight <- someNatVal <$> choose 2 15

30+ Just strideWidth <- someNatVal <$> choose 2 15

3631

3732 case (kernelHeight, kernelWidth, strideHeight, strideWidth) of

3833 (SomeNat (_ :: Proxy kh), SomeNat (_ :: Proxy kw), SomeNat (_ :: Proxy sh), SomeNat (_ :: Proxy sw)) ->

test/Test/Grenade/Network.hs+33−37

@@ -14,12 +14,17 @@ import Control.Monad ( guard )

1414import Control.Monad.ST ( runST )

1515

1616import Data.Constraint

17+#if __GLASGOW_HASKELL__ < 800

18+import Data.Proxy

19+#endif

1720import qualified Data.Vector.Storable as VS

1821import qualified Data.Vector.Storable.Mutable as VS ( write )

1922import Data.Singletons

2023import Data.Singletons.Prelude.List

2124import Data.Singletons.TypeLits

2225

26+-- import Data.Type.Equality

27+

2328import Hedgehog

2429import qualified Hedgehog.Gen as Gen

2530import Hedgehog.Internal.Source

@@ -27,15 +32,7 @@ import Hedgehog.Internal.Property ( failWith )

2732

2833import Grenade

2934

30−#if MIN_VERSION_base(4,11,0)

31−import GHC.TypeLits hiding (natVal)

32−#else

3335import GHC.TypeLits

34−#endif

35−#if MIN_VERSION_base(4,9,0)

36−import Data.Kind (Type)

37−#endif

38−

3936import GHC.TypeLits.Witnesses

4037import Test.Hedgehog.Compat

4138import Test.Hedgehog.TypeLits

@@ -46,7 +43,7 @@ import Numeric.LinearAlgebra ( flatten )

4643import Numeric.LinearAlgebra.Static ( extract, norm_Inf )

4744import Unsafe.Coerce

4845

49−data SomeNetwork :: Type where

46+data SomeNetwork :: * where

5047 SomeNetwork :: ( SingI shapes, SingI (Head shapes), SingI (Last shapes), Show (Network layers shapes) ) => Network layers shapes -> SomeNetwork

5148

5249instance Show SomeNetwork where

@@ -77,9 +74,8 @@ genNetwork =

7774 , pure (SomeNetwork (Elu :~> rest :: Network ( Elu ': layers ) ( h ': h ': hs )))

7875 , pure (SomeNetwork (Softmax :~> rest :: Network ( Softmax ': layers ) ( h ': h ': hs )))

7976 , do -- Reshape to two dimensions

80− let divisors :: Integer -> [Integer]

81− divisors n = 1 : [x | x <- [2..(n-1)], n `rem` x == 0]

82− let len = fromIntegral $ natVal l

77+ let divisors n = 1 : [x | x <- [2..(n-1)], n `rem` x == 0]

78+ let len = natVal l

8379 rs <- Gen.element $ divisors len

8480 let cs = len `quot` rs

8581 case ( someNatVal rs, someNatVal cs, someNatVal len ) of

@@ -100,8 +96,8 @@ genNetwork =

10096 , do -- Build a convolution layer with one filter output

10197 -- Figure out some kernel sizes which work for this layer

10298 -- There must be a better way than this...

103− let output_r = fromIntegral $ natVal r

104− let output_c = fromIntegral $ natVal c

99+ let output_r = natVal r

100+ let output_c = natVal c

105101

106102 let ok extent kernel = [stride | stride <- [ 1 .. extent ], (extent - kernel) `mod` stride == 0]

107103

@@ -140,8 +136,8 @@ genNetwork =

140136 , do -- Build a convolution layer with one filter output

141137 -- Figure out some kernel sizes which work for this layer

142138 -- There must be a better way than this...

143− let output_r = fromIntegral $ natVal r

144− let output_c = fromIntegral $ natVal c

139+ let output_r = natVal r

140+ let output_c = natVal c

145141

146142 let ok extent kernel = [stride | stride <- [ 1 .. extent ], (extent - kernel) `mod` stride == 0]

147143

@@ -182,8 +178,8 @@ genNetwork =

182178 pure (SomeNetwork (conv :~> rest :: Network ( Convolution channels 1 kernelRows kernelCols strideRows strideCols ': layers ) ( ('D3 inRows inCols channels) ': h ': hs )))

183179 _ -> Gen.discard -- Can't occur

184180 , do -- Build a Pooling layer

185− let output_r = fromIntegral $ natVal r

186− let output_c = fromIntegral $ natVal c

181+ let output_r = natVal r

182+ let output_c = natVal c

187183

188184 let ok extent kernel = [stride | stride <- [ 1 .. extent ], (extent - kernel) `mod` stride == 0]

189185

@@ -219,8 +215,8 @@ genNetwork =

219215 pure (SomeNetwork (Pooling :~> rest :: Network ( Pooling kernelRows kernelCols strideRows strideCols ': layers ) ( ('D2 inRows inCols) ': h ': hs )))

220216 _ -> Gen.discard -- Can't occur

221217 , do -- Build a Pad layer

222− let output_r = fromIntegral $ natVal r

223− let output_c = fromIntegral $ natVal c

218+ let output_r = natVal r

219+ let output_c = natVal c

224220

225221 pad_left <- choose 0 (output_r - 1)

226222 pad_right <- choose 0 (output_r - 1 - pad_left)

@@ -246,8 +242,8 @@ genNetwork =

246242 pure (SomeNetwork (Pad :~> rest :: Network ( Pad padLeft padTop padRight padBottom ': layers ) ( ('D2 inputRows inputColumns) ': h ': hs )))

247243 _ -> Gen.discard -- Can't occur

248244 , do -- Build a Crop layer

249− let output_r = fromIntegral $ natVal r

250− let output_c = fromIntegral $ natVal c

245+ let output_r = natVal r

246+ let output_c = natVal c

251247

252248 crop_left <- choose 0 10

253249 crop_right <- choose 0 10

@@ -279,9 +275,9 @@ genNetwork =

279275 , do -- Build a convolution layer with one filter output

280276 -- Figure out some kernel sizes which work for this layer

281277 -- There must be a better way than this...

282− let output_r = fromIntegral $ natVal r

283− let output_c = fromIntegral $ natVal c

284− let output_f = fromIntegral $ natVal f

278+ let output_r = natVal r

279+ let output_c = natVal c

280+ let output_f = natVal f

285281

286282 let ok extent kernel = [stride | stride <- [ 1 .. extent ], (extent - kernel) `mod` stride == 0]

287283

@@ -322,9 +318,9 @@ genNetwork =

322318 pure (SomeNetwork (conv :~> rest :: Network ( Convolution channels filters kernelRows kernelCols strideRows strideCols ': layers ) ( ('D3 inRows inCols channels) ': h ': hs )))

323319 _ -> Gen.discard -- Can't occur

324320 , do -- Build a Pooling layer

325− let output_r = fromIntegral $ natVal r

326− let output_c = fromIntegral $ natVal c

327− let output_f = fromIntegral $ natVal f

321+ let output_r = natVal r

322+ let output_c = natVal c

323+ let output_f = natVal f

328324

329325 let ok extent kernel = [stride | stride <- [ 1 .. extent ], (extent - kernel) `mod` stride == 0]

330326

@@ -363,9 +359,9 @@ genNetwork =

363359 pure (SomeNetwork (Pooling :~> rest :: Network ( Pooling kernelRows kernelCols strideRows strideCols ': layers ) ( ('D3 inRows inCols filters) ': h ': hs )))

364360 _ -> Gen.discard -- Can't occur

365361 , do -- Build a Pad layer

366− let output_r = fromIntegral $ natVal r

367− let output_c = fromIntegral $ natVal c

368− let output_f = fromIntegral $ natVal f

362+ let output_r = natVal r

363+ let output_c = natVal c

364+ let output_f = natVal f

369365

370366 pad_left <- choose 0 (output_r - 1)

371367 pad_right <- choose 0 (output_r - 1 - pad_left)

@@ -393,9 +389,9 @@ genNetwork =

393389 pure (SomeNetwork (Pad :~> rest :: Network ( Pad padLeft padTop padRight padBottom ': layers ) ( ('D3 inputRows inputColumns filters) ': h ': hs )))

394390 _ -> Gen.discard -- Can't occur

395391 , do -- Build a Crop layer

396− let output_r = fromIntegral $ natVal r

397− let output_c = fromIntegral $ natVal c

398− let output_f = fromIntegral $ natVal f

392+ let output_r = natVal r

393+ let output_c = natVal c

394+ let output_f = natVal f

399395

400396 crop_left <- choose 0 10

401397 crop_right <- choose 0 10

@@ -448,7 +444,7 @@ oneUp =

448444 D1Sing SNat ->

449445 let x = 0 :: S ( shape )

450446 in case x of

451− ( S1D x' ) -> do

447+ ( S1D x' ) -> do

452448 let ex = extract x'

453449 let len = VS.length ex

454450 ix <- choose 0 (len - 1)

@@ -460,7 +456,7 @@ oneUp =

460456 D2Sing SNat SNat ->

461457 let x = 0 :: S ( shape )

462458 in case x of

463− ( S2D x' ) -> do

459+ ( S2D x' ) -> do

464460 let ex = flatten ( extract x' )

465461 let len = VS.length ex

466462 ix <- choose 0 (len - 1)

@@ -472,7 +468,7 @@ oneUp =

472468 D3Sing SNat SNat SNat ->

473469 let x = 0 :: S ( shape )

474470 in case x of

475− ( S3D x' ) -> do

471+ ( S3D x' ) -> do

476472 let ex = flatten ( extract x' )

477473 let len = VS.length ex

478474 ix <- choose 0 (len - 1)

test/Test/Grenade/Recurrent/Layers/LSTM.hs+2−2

@@ -121,9 +121,9 @@ prop_lstm_reference_backwards_cell =

121121 , showPretty x

122122 , showPretty y

123123 ]

124− Just differ ->

124+ Just diff ->

125125 withFrozenCallStack $

126− failWith (Just $ Diff "Failed (" "- lhs" "~/~" "+ rhs" ")" differ) ""

126+ failWith (Just $ Diff "Failed (" "- lhs" "~/~" "+ rhs" ")" diff) ""

127127infix 4 ~~~

128128

129129tests :: IO Bool

test/Test/Grenade/Recurrent/Layers/LSTM/Reference.hs+21−25

@@ -6,6 +6,7 @@

66{-# LANGUAGE DeriveFunctor #-}

77{-# LANGUAGE DeriveFoldable #-}

88{-# LANGUAGE DeriveTraversable #-}

9+{-# LANGUAGE RecordWildCards #-}

910{-# LANGUAGE FlexibleContexts #-}

1011{-# LANGUAGE RankNTypes #-}

1112

@@ -14,13 +15,9 @@ module Test.Grenade.Recurrent.Layers.LSTM.Reference where

1415

1516import Data.Reflection

1617import Numeric.AD.Mode.Reverse

17−import Numeric.AD.Internal.Reverse (Tape)

18+import Numeric.AD.Internal.Reverse ( Tape )

1819

19−import GHC.TypeLits (KnownNat)

20−

21−import Grenade.Recurrent.Layers.LSTM (LSTMWeights (..))

2220import qualified Grenade.Recurrent.Layers.LSTM as LSTM

23−

2421import qualified Numeric.LinearAlgebra.Static as S

2522import qualified Numeric.LinearAlgebra as H

2623

@@ -57,32 +54,31 @@ data RefLSTM a = RefLSTM

5754 , refLstmBc :: Vector a -- Bias Cell (b_c)

5855 } deriving (Functor, Foldable, Traversable, Eq, Show)

5956

60−lstmToReference :: (KnownNat a, KnownNat b) => LSTM.LSTMWeights a b -> RefLSTM Double

61−lstmToReference lw =

62− RefLSTM

63− { refLstmWf = Matrix . H.toLists . S.extract $ lstmWf lw -- Weight Forget (W_f)

64− , refLstmUf = Matrix . H.toLists . S.extract $ lstmUf lw -- Cell State Forget (U_f)

65− , refLstmBf = Vector . H.toList . S.extract $ lstmBf lw -- Bias Forget (b_f)

66− , refLstmWi = Matrix . H.toLists . S.extract $ lstmWi lw -- Weight Input (W_i)

67− , refLstmUi = Matrix . H.toLists . S.extract $ lstmUi lw -- Cell State Input (U_i)

68− , refLstmBi = Vector . H.toList . S.extract $ lstmBi lw -- Bias Input (b_i)

69− , refLstmWo = Matrix . H.toLists . S.extract $ lstmWo lw -- Weight Output (W_o)

70− , refLstmUo = Matrix . H.toLists . S.extract $ lstmUo lw -- Cell State Output (U_o)

71− , refLstmBo = Vector . H.toList . S.extract $ lstmBo lw -- Bias Output (b_o)

72− , refLstmWc = Matrix . H.toLists . S.extract $ lstmWc lw -- Weight Cell (W_c)

73− , refLstmBc = Vector . H.toList . S.extract $ lstmBc lw -- Bias Cell (b_c)

74− }

57+lstmToReference :: LSTM.LSTMWeights a b -> RefLSTM Double

58+lstmToReference LSTM.LSTMWeights {..} =

59+ let refLstmWf = Matrix . H.toLists . S.extract $ lstmWf -- Weight Forget (W_f)

60+ refLstmUf = Matrix . H.toLists . S.extract $ lstmUf -- Cell State Forget (U_f)

61+ refLstmBf = Vector . H.toList . S.extract $ lstmBf -- Bias Forget (b_f)

62+ refLstmWi = Matrix . H.toLists . S.extract $ lstmWi -- Weight Input (W_i)

63+ refLstmUi = Matrix . H.toLists . S.extract $ lstmUi -- Cell State Input (U_i)

64+ refLstmBi = Vector . H.toList . S.extract $ lstmBi -- Bias Input (b_i)

65+ refLstmWo = Matrix . H.toLists . S.extract $ lstmWo -- Weight Output (W_o)

66+ refLstmUo = Matrix . H.toLists . S.extract $ lstmUo -- Cell State Output (U_o)

67+ refLstmBo = Vector . H.toList . S.extract $ lstmBo -- Bias Output (b_o)

68+ refLstmWc = Matrix . H.toLists . S.extract $ lstmWc -- Weight Cell (W_c)

69+ refLstmBc = Vector . H.toList . S.extract $ lstmBc -- Bias Cell (b_c)

70+ in RefLSTM {..}

7571

7672runLSTM :: Floating a => RefLSTM a -> Vector a -> Vector a -> (Vector a, Vector a)

77−runLSTM rl cell input =

73+runLSTM RefLSTM {..} cell input =

7874 let -- Forget state vector

79− f_t = sigmoid $ refLstmBf rl #+ refLstmWf rl #> input #+ refLstmUf rl #> cell

75+ f_t = sigmoid $ refLstmBf #+ refLstmWf #> input #+ refLstmUf #> cell

8076 -- Input state vector

81− i_t = sigmoid $ refLstmBi rl #+ refLstmWi rl #> input #+ refLstmUi rl #> cell

77+ i_t = sigmoid $ refLstmBi #+ refLstmWi #> input #+ refLstmUi #> cell

8278 -- Output state vector

83− o_t = sigmoid $ refLstmBo rl #+ refLstmWo rl #> input #+ refLstmUo rl #> cell

79+ o_t = sigmoid $ refLstmBo #+ refLstmWo #> input #+ refLstmUo #> cell

8480 -- Cell input state vector

85− c_x = fmap tanh $ refLstmBc rl #+ refLstmWc rl #> input

81+ c_x = fmap tanh $ refLstmBc #+ refLstmWc #> input

8682 -- Cell state

8783 c_t = f_t #* cell #+ i_t #* c_x

8884 -- Output (it's sometimes recommended to use tanh c_t)

test/Test/Hedgehog/Hmatrix.hs+2−3

@@ -1,8 +1,7 @@

1−{-# LANGUAGE DataKinds #-}

2−{-# LANGUAGE KindSignatures #-}

3−{-# LANGUAGE GADTs #-}

41{-# LANGUAGE RankNTypes #-}

2+{-# LANGUAGE DataKinds #-}

53{-# LANGUAGE ScopedTypeVariables #-}

4+{-# LANGUAGE GADTs #-}

65

76module Test.Hedgehog.Hmatrix where

87

test/Test/Hedgehog/TypeLits.hs+19−13

@@ -7,27 +7,33 @@

77{-# LANGUAGE GADTs #-}

88module Test.Hedgehog.TypeLits where

99

10−import Data.Constraint (Dict (..))

11−import Data.Singletons (Proxy (..), Sing (..), SomeSing (..), sing)

12−#if MIN_VERSION_singletons(2,6,0)

13−import Data.Singletons.TypeLits (SNat (..))

10+import Data.Constraint

11+#if __GLASGOW_HASKELL__ < 800

12+import Data.Proxy

1413#endif

14+import Data.Singletons

1515

1616import Hedgehog (Gen)

1717import qualified Hedgehog.Gen as Gen

1818

1919import Grenade

2020

21−import GHC.TypeLits (SomeNat (..), natVal, someNatVal)

22−import GHC.TypeLits.Witnesses ((%*), natDict)

23−import Test.Hedgehog.Compat (choose)

21+import GHC.TypeLits

22+import GHC.TypeLits.Witnesses

23+import Test.Hedgehog.Compat

2424

2525genNat :: Gen SomeNat

2626genNat = do

27− ~(Just n) <- someNatVal <$> choose 1 10

27+ Just n <- someNatVal <$> choose 1 10

2828 return n

2929

30−genShape :: Gen (SomeSing Shape)

30+#if __GLASGOW_HASKELL__ < 800

31+type Shape' = ('KProxy :: KProxy Shape)

32+#else

33+type Shape' = Shape

34+#endif

35+

36+genShape :: Gen (SomeSing Shape')

3137genShape

3238 = Gen.choice [

3339 genD1

@@ -35,20 +41,20 @@ genShape

3541 , genD3

3642 ]

3743

38−genD1 :: Gen (SomeSing Shape)

44+genD1 :: Gen (SomeSing Shape')

3945genD1 = do

4046 n <- genNat

4147 return $ case n of

4248 SomeNat (_ :: Proxy x) -> SomeSing (sing :: Sing ('D1 x))

4349

44−genD2 :: Gen (SomeSing Shape)

50+genD2 :: Gen (SomeSing Shape')

4551genD2 = do

4652 n <- genNat

4753 m <- genNat

4854 return $ case (n, m) of

4955 (SomeNat (_ :: Proxy x), SomeNat (_ :: Proxy y)) -> SomeSing (sing :: Sing ('D2 x y))

5056

51−genD3 :: Gen (SomeSing Shape)

57+genD3 :: Gen (SomeSing Shape')

5258genD3 = do

5359 n <- genNat

5460 m <- genNat

@@ -58,7 +64,7 @@ genD3 = do

5864 case natDict px %* natDict pz of

5965 Dict -> SomeSing (sing :: Sing ('D3 x y z))

6066

61−rss :: SomeSing Shape -> String

67+rss :: SomeSing Shape' -> String

6268rss (SomeSing (r :: Sing s)) = case r of

6369 (D1Sing a@SNat) -> "D1 " ++ show (natVal a)

6470 (D2Sing a@SNat b@SNat) -> "D2 " ++ show (natVal a) ++ " " ++ show (natVal b)