d118c5e · grenade · git.iko.soy

git.iko.soy/grenade

Added NFData instances for networks and layers

commitd118c5e6f384fdb67c3e72f18ed092a3d3d20895 authoriko <ilyakooo0@gmail.com> dateTue 5 Apr 2022, 08:26 (5 years ago) parent5206c95

24 files changed+117−4

compare with master

unifiedsplit

src/Grenade/Core/Layer.hs+2−1

@@ -48,6 +48,7 @@ import Data.Kind (Type)

4848

4949import Grenade.Core.Shape

5050import Grenade.Core.LearningParameters

51+import Control.DeepSeq (NFData)

5152

5253-- | Class for updating a layer. All layers implement this, as it

5354-- describes how to create and update the layer.

@@ -73,7 +74,7 @@ class UpdateLayer x where

7374-- need to implement it for all shapes, only ones which are

7475-- appropriate.

7576--

76−class UpdateLayer x => Layer x (i :: Shape) (o :: Shape) where

77+class (UpdateLayer x, NFData x) => Layer x (i :: Shape) (o :: Shape) where

7778 -- | The Wengert tape for this layer. Includes all that is required

7879 -- to generate the back propagated gradients efficiently. As a

7980 -- default, `S i` is fine.

src/Grenade/Core/Network.hs+8−1

@@ -44,6 +44,7 @@ import Data.Kind (Type)

4444import Grenade.Core.Layer

4545import Grenade.Core.LearningParameters

4646import Grenade.Core.Shape

47+import Control.DeepSeq

4748

4849-- | Type of a network.

4950--

@@ -68,6 +69,12 @@ instance Show (Network '[] '[i]) where

6869instance (Show x, Show (Network xs rs)) => Show (Network (x ': xs) (i ': rs)) where

6970 show (x :~> xs) = show x ++ "\n~>\n" ++ show xs

7071

72+instance NFData (Network '[] '[i]) where

73+ rnf NNil = ()

74+

75+instance NFData (Network xs rs) => NFData (Network (x ': xs) (i ': rs)) where

76+ rnf (x :~> xs) = x `deepseq` xs `deepseq` ()

77+

7178-- | Gradient of a network.

7279--

7380-- Parameterised on the layers of the network.

@@ -191,7 +198,7 @@ instance CreatableNetwork sublayers subshapes => UpdateLayer (Network sublayers

191198-- | Ultimate composition.

192199--

193200-- This allows a complete network to be treated as a layer in a larger network.

194−instance (CreatableNetwork sublayers subshapes, i ~ (Head subshapes), o ~ (Last subshapes)) => Layer (Network sublayers subshapes) i o where

201+instance (CreatableNetwork sublayers subshapes, i ~ (Head subshapes), o ~ (Last subshapes), NFData (Network sublayers subshapes)) => Layer (Network sublayers subshapes) i o where

195202 type Tape (Network sublayers subshapes) i o = Tapes sublayers subshapes

196203 runForwards = runNetwork

197204 runBackwards = runGradient

src/Grenade/Layers/Concat.hs+4

@@ -35,6 +35,7 @@ import Data.Kind (Type)

3535import Grenade.Core

3636

3737import Numeric.LinearAlgebra.Static ( row, (===), splitRows, unrow, (#), split, R )

38+import Control.DeepSeq

3839

3940-- | A Concatentating Layer.

4041--

@@ -52,6 +53,9 @@ import Numeric.LinearAlgebra.Static ( row, (===), splitRows, unrow, (#

5253data Concat :: Shape -> Type -> Shape -> Type -> Type where

5354 Concat :: x -> y -> Concat m x n y

5455

56+instance (NFData x, NFData y) => NFData (Concat m x n y) where

57+ rnf (Concat x y) = rnf x `deepseq` rnf y `deepseq` ()

58+

5559instance (Show x, Show y) => Show (Concat m x n y) where

5660 show (Concat x y) = "Concat\n" ++ show x ++ "\n" ++ show y

5761

src/Grenade/Layers/Convolution.hs+4

@@ -46,6 +46,7 @@ import Numeric.LinearAlgebra.Static hiding ((|||), build, toRows)

4646import Grenade.Core

4747import Grenade.Layers.Internal.Convolution

4848import Grenade.Layers.Internal.Update

49+import Control.DeepSeq

4950

5051-- | A convolution layer for a neural network.

5152-- This uses the im2col convolution trick popularised by Caffe, which essentially turns the

@@ -113,6 +114,9 @@ instance Show (Convolution c f k k' s s') where

113114 px = (fmap . fmap . fmap) render ms

114115 in unlines $ foldl1 (zipWith (\a' b' -> a' ++ " | " ++ b')) $ px

115116

117+instance NFData (Convolution c f k k' s s') where

118+ rnf (Convolution a b) = rnf a `deepseq` rnf b `deepseq` ()

119+

116120randomConvolution :: ( MonadRandom m

117121 , KnownNat channels

118122 , KnownNat filters

src/Grenade/Layers/Crop.hs+4

@@ -37,6 +37,7 @@ import Grenade.Layers.Internal.Pad

3737

3838import Numeric.LinearAlgebra (konst, subMatrix, diagBlock)

3939import Numeric.LinearAlgebra.Static (extract, create)

40+import Control.DeepSeq

4041

4142-- | A cropping layer for a neural network.

4243data Crop :: Nat

@@ -45,6 +46,9 @@ data Crop :: Nat

4546 -> Nat -> Type where

4647 Crop :: Crop cropLeft cropTop cropRight cropBottom

4748

49+instance NFData (Crop cropLeft cropTop cropRight cropBottom) where

50+ rnf Crop = ()

51+

4852instance Show (Crop cropLeft cropTop cropRight cropBottom) where

4953 show Crop = "Crop"

5054

src/Grenade/Layers/Deconvolution.hs+4

@@ -50,6 +50,7 @@ import Numeric.LinearAlgebra.Static hiding ((|||), build, toRows)

5050import Grenade.Core

5151import Grenade.Layers.Internal.Convolution

5252import Grenade.Layers.Internal.Update

53+import Control.DeepSeq

5354

5455-- | A Deconvolution layer for a neural network.

5556-- This uses the im2col Convolution trick popularised by Caffe.

@@ -94,6 +95,9 @@ data Deconvolution' :: Nat -- Number of channels, for the first layer this could

9495 => !(L kernelFlattened channels) -- The kernel filter gradient

9596 -> Deconvolution' channels filters kernelRows kernelColumns strideRows strideColumns

9697

98+instance NFData (Deconvolution c f k k' s s') where

99+ rnf (Deconvolution a b) = rnf a `deepseq` rnf b `deepseq` ()

100+

97101instance Show (Deconvolution c f k k' s s') where

98102 show (Deconvolution a _) = renderConv a

99103 where

src/Grenade/Layers/Dropout.hs+5−1

@@ -11,6 +11,7 @@ import Control.Monad.Random hiding (fromList)

1111

1212import GHC.TypeLits

1313import Grenade.Core

14+import Control.DeepSeq

1415

1516-- Dropout layer help to reduce overfitting.

1617-- Idea here is that the vector is a shape of 1s and 0s, which we multiply the input by.

@@ -20,7 +21,10 @@ import Grenade.Core

2021data Dropout = Dropout {

2122 dropoutRate :: Double

2223 , dropoutSeed :: Int

23− } deriving Show

24+ } deriving (Show)

25+

26+instance NFData Dropout where

27+ rnf (Dropout r s) = rnf r `deepseq` rnf s `deepseq` ()

2428

2529instance UpdateLayer Dropout where

2630 type Gradient Dropout = ()

src/Grenade/Layers/Elu.hs+4

@@ -19,6 +19,7 @@ import GHC.TypeLits

1919import Grenade.Core

2020

2121import qualified Numeric.LinearAlgebra.Static as LAS

22+import Control.DeepSeq

2223

2324-- | An exponential linear unit.

2425-- A layer which can act between any shape of the same dimension, acting as a

@@ -26,6 +27,9 @@ import qualified Numeric.LinearAlgebra.Static as LAS

2627data Elu = Elu

2728 deriving Show

2829

30+instance NFData Elu where

31+ rnf Elu = ()

32+

2933instance UpdateLayer Elu where

3034 type Gradient Elu = ()

3135 runUpdate _ _ _ = Elu

src/Grenade/Layers/FullyConnected.hs+7

@@ -21,6 +21,7 @@ import Numeric.LinearAlgebra.Static

2121import Grenade.Core

2222

2323import Grenade.Layers.Internal.Update

24+import Control.DeepSeq

2425

2526-- | A basic fully connected (or inner product) neural network layer.

2627data FullyConnected i o = FullyConnected

@@ -31,6 +32,12 @@ data FullyConnected' i o = FullyConnected'

3132 !(R o) -- Bias

3233 !(L o i) -- Activations

3334

35+instance NFData (FullyConnected' i o) where

36+ rnf (FullyConnected' a b) = a `deepseq` b `deepseq` ()

37+

38+instance NFData (FullyConnected i o) where

39+ rnf (FullyConnected a b) = a `deepseq` b `deepseq` ()

40+

3441instance Show (FullyConnected i o) where

3542 show FullyConnected {} = "FullyConnected"

3643

src/Grenade/Layers/Logit.hs+4

@@ -19,6 +19,7 @@ import Data.Serialize

1919import Data.Singletons

2020

2121import Grenade.Core

22+import Control.DeepSeq

2223

2324-- | A Logit layer.

2425--

@@ -28,6 +29,9 @@ import Grenade.Core

2829data Logit = Logit

2930 deriving Show

3031

32+instance NFData Logit where

33+ rnf Logit = ()

34+

3135instance UpdateLayer Logit where

3236 type Gradient Logit = ()

3337 runUpdate _ _ _ = Logit

src/Grenade/Layers/Merge.hs+4

@@ -29,6 +29,7 @@ import Data.Kind (Type)

2929#endif

3030

3131import Grenade.Core

32+import Control.DeepSeq

3233

3334-- | A Merging layer.

3435--

@@ -37,6 +38,9 @@ import Grenade.Core

3738data Merge :: Type -> Type -> Type where

3839 Merge :: x -> y -> Merge x y

3940

41+instance (NFData x, NFData y) => NFData (Merge x y) where

42+ rnf (Merge x y) = rnf x `deepseq` rnf y `deepseq` ()

43+

4044instance (Show x, Show y) => Show (Merge x y) where

4145 show (Merge x y) = "Merge\n" ++ show x ++ "\n" ++ show y

4246

src/Grenade/Layers/Pad.hs+4

@@ -37,6 +37,7 @@ import Grenade.Layers.Internal.Pad

3737

3838import Numeric.LinearAlgebra (konst, subMatrix, diagBlock)

3939import Numeric.LinearAlgebra.Static (extract, create)

40+import Control.DeepSeq

4041

4142-- | A padding layer for a neural network.

4243--

@@ -47,6 +48,9 @@ data Pad :: Nat

4748 -> Nat -> Type where

4849 Pad :: Pad padLeft padTop padRight padBottom

4950

51+instance NFData (Pad padLeft padTop padRight padBottom) where

52+ rnf Pad = ()

53+

5054instance Show (Pad padLeft padTop padRight padBottom) where

5155 show Pad = "Pad"

5256

src/Grenade/Layers/Pooling.hs+4

@@ -37,6 +37,7 @@ import Grenade.Core

3737import Grenade.Layers.Internal.Pooling

3838

3939import Numeric.LinearAlgebra.Static as LAS hiding ((|||), build, toRows)

40+import Control.DeepSeq

4041

4142-- | A pooling layer for a neural network.

4243--

@@ -49,6 +50,9 @@ import Numeric.LinearAlgebra.Static as LAS hiding ((|||), build, toRow

4950data Pooling :: Nat -> Nat -> Nat -> Nat -> Type where

5051 Pooling :: Pooling kernelRows kernelColumns strideRows strideColumns

5152

53+instance NFData (Pooling k k' s s') where

54+ rnf Pooling = ()

55+

5256instance Show (Pooling k k' s s') where

5357 show Pooling = "Pooling"

5458

src/Grenade/Layers/Relu.hs+4

@@ -19,6 +19,7 @@ import GHC.TypeLits

1919import Grenade.Core

2020

2121import qualified Numeric.LinearAlgebra.Static as LAS

22+import Control.DeepSeq

2223

2324-- | A rectifying linear unit.

2425-- A layer which can act between any shape of the same dimension, acting as a

@@ -26,6 +27,9 @@ import qualified Numeric.LinearAlgebra.Static as LAS

2627data Relu = Relu

2728 deriving Show

2829

30+instance NFData Relu where

31+ rnf Relu = ()

32+

2933instance UpdateLayer Relu where

3034 type Gradient Relu = ()

3135 runUpdate _ _ _ = Relu

src/Grenade/Layers/Reshape.hs+4

@@ -24,6 +24,7 @@ import Numeric.LinearAlgebra.Static

2424import Numeric.LinearAlgebra.Data as LA ( flatten )

2525

2626import Grenade.Core

27+import Control.DeepSeq

2728

2829-- | Reshape Layer

2930--

@@ -36,6 +37,9 @@ import Grenade.Core

3637data Reshape = Reshape

3738 deriving Show

3839

40+instance NFData Reshape where

41+ rnf Reshape = ()

42+

3943instance UpdateLayer Reshape where

4044 type Gradient Reshape = ()

4145 runUpdate _ _ _ = Reshape

src/Grenade/Layers/Sinusoid.hs+4

@@ -18,12 +18,16 @@ import Data.Serialize

1818import Data.Singletons

1919

2020import Grenade.Core

21+import Control.DeepSeq

2122

2223-- | A Sinusoid layer.

2324-- A layer which can act between any shape of the same dimension, performing a sin function.

2425data Sinusoid = Sinusoid

2526 deriving Show

2627

28+instance NFData Sinusoid where

29+ rnf Sinusoid = ()

30+

2731instance UpdateLayer Sinusoid where

2832 type Gradient Sinusoid = ()

2933 runUpdate _ _ _ = Sinusoid

src/Grenade/Layers/Softmax.hs+4

@@ -22,6 +22,7 @@ import GHC.TypeLits

2222import Grenade.Core

2323

2424import Numeric.LinearAlgebra.Static as LAS

25+import Control.DeepSeq

2526

2627-- | A Softmax layer

2728--

@@ -33,6 +34,9 @@ import Numeric.LinearAlgebra.Static as LAS

3334data Softmax = Softmax

3435 deriving Show

3536

37+instance NFData Softmax where

38+ rnf Softmax = ()

39+

3640instance UpdateLayer Softmax where

3741 type Gradient Softmax = ()

3842 runUpdate _ _ _ = Softmax

src/Grenade/Layers/Tanh.hs+4

@@ -18,12 +18,16 @@ import Data.Serialize

1818import Data.Singletons

1919

2020import Grenade.Core

21+import Control.DeepSeq

2122

2223-- | A Tanh layer.

2324-- A layer which can act between any shape of the same dimension, performing a tanh function.

2425data Tanh = Tanh

2526 deriving Show

2627

28+instance NFData Tanh where

29+ rnf Tanh = ()

30+

2731instance UpdateLayer Tanh where

2832 type Gradient Tanh = ()

2933 runUpdate _ _ _ = Tanh

src/Grenade/Layers/Trivial.hs+4

@@ -17,6 +17,7 @@ module Grenade.Layers.Trivial (

1717import Data.Serialize

1818

1919import Grenade.Core

20+import Control.DeepSeq

2021

2122-- | A Trivial layer.

2223--

@@ -25,6 +26,9 @@ import Grenade.Core

2526data Trivial = Trivial

2627 deriving Show

2728

29+instance NFData Trivial where

30+ rnf Trivial = ()

31+

2832instance Serialize Trivial where

2933 put _ = return ()

3034 get = return Trivial

src/Grenade/Recurrent/Core/Layer.hs+2−1

@@ -14,6 +14,7 @@ import Data.Kind (Type)

1414#endif

1515

1616import Grenade.Core

17+import Control.DeepSeq (NFData)

1718

1819-- | Class for a recurrent layer.

1920-- It's quite similar to a normal layer but for the input and output

@@ -22,7 +23,7 @@ class UpdateLayer x => RecurrentUpdateLayer x where

2223 -- | Shape of data that is passed between each subsequent run of the layer

2324 type RecurrentShape x :: Type

2425

25−class (RecurrentUpdateLayer x, Num (RecurrentShape x)) => RecurrentLayer x (i :: Shape) (o :: Shape) where

26+class (RecurrentUpdateLayer x, Num (RecurrentShape x), NFData x) => RecurrentLayer x (i :: Shape) (o :: Shape) where

2627 -- | Wengert Tape

2728 type RecTape x i o :: Type

2829 -- | Used in training and scoring. Take the input from the previous

src/Grenade/Recurrent/Core/Network.hs+8

@@ -39,6 +39,7 @@ import Data.Kind (Type)

3939

4040import Grenade.Core

4141import Grenade.Recurrent.Core.Layer

42+import Control.DeepSeq

4243

4344-- | Witness type to say indicate we're building up with a normal feed

4445-- forward layer.

@@ -198,6 +199,12 @@ applyRecurrentUpdate rate (layer :~@> rest) (gradient ://> grest)

198199applyRecurrentUpdate _ RNil RGNil

199200 = RNil

200201

202+instance NFData (RecurrentNetwork '[] '[i]) where

203+ rnf RNil = ()

204+

205+instance NFData (RecurrentNetwork xs rs) => NFData (RecurrentNetwork (x ': xs) (i ': rs)) where

206+ rnf (layer :~~> rest) = rnf layer `deepseq` rnf rest `deepseq` ()

207+ rnf (layer :~@> rest) = rnf layer `deepseq` rnf rest `deepseq` ()

201208

202209instance Show (RecurrentNetwork '[] '[i]) where

203210 show RNil = "NNil"

@@ -325,6 +332,7 @@ instance CreatableRecurrent sublayers subshapes => RecurrentUpdateLayer (Recurre

325332instance ( CreatableRecurrent sublayers subshapes

326333 , i ~ (Head subshapes), o ~ (Last subshapes)

327334 , Num (RecurrentShape (RecurrentNetwork sublayers subshapes))

335+ , NFData (RecurrentNetwork sublayers subshapes)

328336 ) => RecurrentLayer (RecurrentNetwork sublayers subshapes) i o where

329337 type RecTape (RecurrentNetwork sublayers subshapes) i o = RecurrentTape sublayers subshapes

330338 runRecurrentForwards = runRecurrent

src/Grenade/Recurrent/Layers/BasicRecurrent.hs+4

@@ -28,6 +28,7 @@ import GHC.TypeLits

2828

2929import Grenade.Core

3030import Grenade.Recurrent.Core

31+import Control.DeepSeq

3132

3233data BasicRecurrent :: Nat -- Input layer size

3334 -> Nat -- Output layer size

@@ -42,6 +43,9 @@ data BasicRecurrent :: Nat -- Input layer size

4243 -> !(L output matrixCols) -- Momentum

4344 -> BasicRecurrent input output

4445

46+instance NFData (BasicRecurrent input output) where

47+ rnf (BasicRecurrent b bm a aM) = rnf b `deepseq` rnf bm `deepseq` rnf a `deepseq` rnf aM `deepseq` ()

48+

4549data BasicRecurrent' :: Nat -- Input layer size

4650 -> Nat -- Output layer size

4751 -> Type where

src/Grenade/Recurrent/Layers/ConcatRecurrent.hs+10

@@ -36,6 +36,7 @@ import Grenade.Core

3636import Grenade.Recurrent.Core

3737

3838import Numeric.LinearAlgebra.Static ( (#), split, R )

39+import Control.DeepSeq

3940

4041-- | A Concatentating Layer.

4142--

@@ -55,6 +56,15 @@ data ConcatRecurrent :: Shape -> Type -> Shape -> Type -> Type where

5556 ConcatRecRight :: x -> y -> ConcatRecurrent m (FeedForward x) n (Recurrent y)

5657 ConcatRecBoth :: x -> y -> ConcatRecurrent m (Recurrent x) n (Recurrent y)

5758

59+instance (NFData x, NFData y) => NFData (ConcatRecurrent m (Recurrent x) n (Recurrent y)) where

60+ rnf (ConcatRecBoth x y) = rnf x `deepseq` rnf y `deepseq` ()

61+

62+instance (NFData x, NFData y) => NFData (ConcatRecurrent m (Recurrent x) n (FeedForward y)) where

63+ rnf (ConcatRecLeft x y) = rnf x `deepseq` rnf y `deepseq` ()

64+

65+instance (NFData x, NFData y) => NFData (ConcatRecurrent m (FeedForward x) n (Recurrent y)) where

66+ rnf (ConcatRecRight x y) = rnf x `deepseq` rnf y `deepseq` ()

67+

5868instance (Show x, Show y) => Show (ConcatRecurrent m (p x) n (q y)) where

5969 show (ConcatRecLeft x y) = "ConcatRecLeft\n" ++ show x ++ "\n" ++ show y

6070 show (ConcatRecRight x y) = "ConcatRecRight\n" ++ show x ++ "\n" ++ show y

src/Grenade/Recurrent/Layers/LSTM.hs+11

@@ -33,6 +33,7 @@ import Numeric.LinearAlgebra.Static

3333import Grenade.Core

3434import Grenade.Recurrent.Core

3535import Grenade.Layers.Internal.Update

36+import Control.DeepSeq

3637

3738

3839-- | Long Short Term Memory Recurrent unit

@@ -64,6 +65,16 @@ data LSTMWeights :: Nat -> Nat -> Type where

6465 , lstmBc :: !(R output) -- Bias Cell (b_c)

6566 } -> LSTMWeights input output

6667

68+instance NFData (LSTMWeights input output) where

69+ rnf (LSTMWeights wf uf bf wi ui bi wo uo bo wc bc) =

70+ rnf wf `deepseq` rnf uf `deepseq` rnf bf `deepseq`

71+ rnf wi `deepseq` rnf ui `deepseq` rnf bi `deepseq`

72+ rnf wo `deepseq` rnf uo `deepseq` rnf bo `deepseq`

73+ rnf wc `deepseq` rnf bc `deepseq` ()

74+

75+instance NFData (LSTM input output) where

76+ rnf (LSTM w m) = rnf w `deepseq` rnf m `deepseq` ()

77+

6778instance Show (LSTM i o) where

6879 show LSTM {} = "LSTM"

6980