git.iko.soy/grenade
on master help sign in
Merge pull request #12 from HuwCampbell/topic/gradients

Topic/gradients

commit3f7cf7f9a6df67dda80129300fbdb2c230bffd90 authorHuw Campbell <huw.campbell@gmail.com> dateWed 7 Dec 2016, 09:16 (10 years ago) parentae4de428cd875a
17 files changed+120−103
compare with master
unifiedsplit
README.md+5−10 main/feedforward.hs+6−10 main/mnist.hs+7−11 src/Grenade/Core/Network.hs+34−10 src/Grenade/Core/Runner.hs+35−23 src/Grenade/Core/Shape.hs+1−18 src/Grenade/Layers/Convolution.hs+12−1 src/Grenade/Layers/Crop.hs+2−5 src/Grenade/Layers/Dropout.hs+1 src/Grenade/Layers/Flatten.hs+2 src/Grenade/Layers/FullyConnected.hs+2 src/Grenade/Layers/Fuse.hs+5−4 src/Grenade/Layers/Logit.hs+1 src/Grenade/Layers/Pad.hs+2−5 src/Grenade/Layers/Pooling.hs+2−5 src/Grenade/Layers/Relu.hs+1 src/Grenade/Layers/Tanh.hs+2−1
README.md+5−10
@@ -14,17 +14,12 @@ Grenade is a dependently typed, practical, and pretty quick neural network libra
1414specifications of complex networks in Haskell.
1515
1616As an example, a network which can achieve less than 1.5% error on MNIST can be specified and
17−initialised with random weights in under 10 lines of code with
17+initialised with random weights in a few lines of code with
1818```haskell
19−randomMnistNet :: MonadRandom m => m (Network '[ 'D2 28 28, 'D3 24 24 10, 'D3 12 12 10, 'D3 12 12 10, 'D3 8 8 16, 'D3 4 4 16, 'D1 256, 'D1 256, 'D1 80, 'D1 80, 'D1 10, 'D1 10])
20−randomMnistNet = do
21− a :: Convolution 1 10 5 5 1 1 <- randomConvolution
22− let b :: Pooling 2 2 2 2 = Pooling
23− c :: Convolution 10 16 5 5 1 1 <- randomConvolution
24− let d :: Pooling 2 2 2 2 = Pooling
25− e :: FullyConnected 256 80 <- randomFullyConnected
26− f :: FullyConnected 80 10 <- randomFullyConnected
27− return $ a :~> b :~> Relu :~> c :~> d :~> FlattenLayer :~> Relu :~> e :~> Logit :~> f :~> O Logit
19+randomMnist :: MonadRandom m
20+ => m (Network '[ Convolution 1 10 5 5 1 1, Pooling 2 2 2 2, Relu, Convolution 10 16 5 5 1 1, Pooling 2 2 2 2, FlattenLayer, Relu, FullyConnected 256 80, Logit, FullyConnected 80 10, Logit]
21+ '[ 'D2 28 28, 'D3 24 24 10, 'D3 12 12 10, 'D3 12 12 10, 'D3 8 8 16, 'D3 4 4 16, 'D1 256, 'D1 256, 'D1 80, 'D1 80, 'D1 10, 'D1 10])
22+randomMnist = randomNetwork
2823```
2924
3025The network can be thought of as a heterogeneous list of layers, and its type signature includes a type
main/feedforward.hs+6−10
@@ -17,7 +17,6 @@ import Options.Applicative
1717
1818import Grenade
1919
20−
2120-- The defininition for our simple feed forward network.
2221-- The type level list represents the shapes passed through the layers. One can see that for this demonstration
2322-- we are using relu, tanh and logit non-linear units, which can be easily subsituted for each other in and out.
@@ -26,12 +25,10 @@ import Grenade
2625-- between the shapes, so inference can't do it all for us.
2726
2827-- With around 100000 examples, this should show two clear circles which have been learned by the network.
29−randomNet :: MonadRandom m => m (Network '[ 'D1 2, 'D1 40, 'D1 40, 'D1 10, 'D1 10, 'D1 1, 'D1 1])
30−randomNet = do
31− a :: FullyConnected 2 40 <- randomFullyConnected
32− b :: FullyConnected 40 10 <- randomFullyConnected
33− c :: FullyConnected 10 1 <- randomFullyConnected
34− return $ a :~> Tanh :~> b :~> Relu :~> c :~> O Logit
28+randomNet :: MonadRandom m
29+ => m (Network '[ FullyConnected 2 40, Tanh, FullyConnected 40 10, Relu, FullyConnected 10 1, Logit ]
30+ '[ 'D1 2, 'D1 40, 'D1 40, 'D1 10, 'D1 10, 'D1 1, 'D1 1])
31+randomNet = randomNetwork
3532
3633netTest :: MonadRandom m => LearningParameters -> Int -> m String
3734netTest rate n = do
@@ -46,7 +43,7 @@ netTest rate n = do
4643
4744 let trained = foldl trainEach net0 (zip inps outs)
4845 let testIns = [ [ (x,y) | x <- [0..50] ]
49− | y <- [0..20] ]
46+ | y <- [0..20] ]
5047
5148 let outMat = fmap (fmap (\(x,y) -> (render . normx) $ runNet trained (S1D' $ SA.vector [x / 25 - 1,y / 10 - 1]))) testIns
5249 return $ unlines outMat
@@ -54,7 +51,7 @@ netTest rate n = do
5451 where
5552 inCircle :: KnownNat n => SA.R n -> (SA.R n, Double) -> Bool
5653 v `inCircle` (o, r) = SA.norm_2 (v - o) <= r
57− trainEach !nt !(i, o) = train rate i o nt
54+ trainEach !nt !(i, o) = train rate nt i o
5855
5956 render n' | n' <= 0.2 = ' '
6057 | n' <= 0.4 = '.'
@@ -65,7 +62,6 @@ netTest rate n = do
6562 normx :: S' ('D1 1) -> Double
6663 normx (S1D' r) = SA.mean r
6764
68−
6965data FeedForwardOpts = FeedForwardOpts Int LearningParameters
7066
7167feedForward' :: Parser FeedForwardOpts
main/mnist.hs+7−11
@@ -30,19 +30,15 @@ import Grenade
3030
3131-- With the mnist data from Kaggle normalised to doubles between 0 and 1, learning rate of 0.01 and 15 iterations,
3232-- this network should get down to about a 1.3% error rate.
33−randomMnistNet :: MonadRandom m => m (Network '[ 'D2 28 28, 'D3 24 24 10, 'D3 12 12 10, 'D3 12 12 10, 'D3 8 8 16, 'D3 4 4 16, 'D1 256, 'D1 256, 'D1 80, 'D1 80, 'D1 10, 'D1 10])
34−randomMnistNet = do
35− a :: Convolution 1 10 5 5 1 1 <- randomConvolution
36− let b :: Pooling 2 2 2 2 = Pooling
37− c :: Convolution 10 16 5 5 1 1 <- randomConvolution
38− let d :: Pooling 2 2 2 2 = Pooling
39− e :: FullyConnected 256 80 <- randomFullyConnected
40− f :: FullyConnected 80 10 <- randomFullyConnected
41− return $ a :~> b :~> Relu :~> c :~> d :~> FlattenLayer :~> Relu :~> e :~> Logit :~> f :~> O Logit
33+randomMnist :: MonadRandom m
34+ => m (Network '[ Convolution 1 10 5 5 1 1, Pooling 2 2 2 2, Relu, Convolution 10 16 5 5 1 1, Pooling 2 2 2 2, FlattenLayer, Relu, FullyConnected 256 80, Logit, FullyConnected 80 10, Logit]
35+ '[ 'D2 28 28, 'D3 24 24 10, 'D3 12 12 10, 'D3 12 12 10, 'D3 8 8 16, 'D3 4 4 16, 'D1 256, 'D1 256, 'D1 80, 'D1 80, 'D1 10, 'D1 10])
36+randomMnist = randomNetwork
37+
4238
4339convTest :: Int -> FilePath -> FilePath -> LearningParameters -> IO ()
4440convTest iterations trainFile validateFile rate = do
45− net0 <- evalRandIO randomMnistNet
41+ net0 <- evalRandIO randomMnist
4642 fT <- T.readFile trainFile
4743 fV <- T.readFile validateFile
4844 let trainRows = traverse (A.parseOnly p) (T.lines fT)
@@ -52,7 +48,7 @@ convTest iterations trainFile validateFile rate = do
5248 err -> print err
5349
5450 where
55− trainEach !rate' !nt !(i, o) = train rate' i o nt
51+ trainEach !rate' !nt !(i, o) = train rate' nt i o
5652
5753 p :: A.Parser (S' ('D2 28 28), S' ('D1 10))
5854 p = do
src/Grenade/Core/Network.hs+34−10
@@ -8,14 +8,20 @@
88{-# LANGUAGE MultiParamTypeClasses #-}
99{-# LANGUAGE FlexibleContexts #-}
1010{-# LANGUAGE FlexibleInstances #-}
11+{-# LANGUAGE LambdaCase #-}
1112
1213module Grenade.Core.Network (
1314 Layer (..)
1415 , Network (..)
1516 , UpdateLayer (..)
1617 , LearningParameters (..)
18+ , Gradients (..)
19+ , CreatableNetwork (..)
1720 ) where
1821
22+import Control.Monad.Random (MonadRandom)
23+
24+
1925import Grenade.Core.Shape
2026
2127data LearningParameters = LearningParameters {
@@ -26,12 +32,14 @@ data LearningParameters = LearningParameters {
2632
2733-- | Class for updating a layer. All layers implement this, and it is
2834-- shape independent.
29−class UpdateLayer x where
35+class Show x => UpdateLayer x where
3036 -- | The type for the gradient for this layer.
3137 -- Unit if there isn't a gradient to pass back.
3238 type Gradient x :: *
3339 -- | Update a layer with its gradient and learning parameters
3440 runUpdate :: LearningParameters -> x -> Gradient x -> x
41+ -- | Create a random layer, many layers will use pure
42+ createRandom :: MonadRandom m => m x
3543
3644-- | Class for a layer. All layers implement this, however, they don't
3745-- need to implement it for all shapes, only ones which are appropriate.
@@ -46,19 +54,35 @@ class UpdateLayer x => Layer x (i :: Shape) (o :: Shape) where
4654 runBackards :: x -> S' i -> S' o -> (Gradient x, S' i)
4755
4856-- | Type of a network.
57+-- The [*] type specifies the types of the layers. This is needed for parallel
58+-- running and being all the gradients beck together.
4959-- The [Shape] type specifies the shapes of data passed between the layers.
5060-- Could be considered to be a heterogeneous list of layers which are able to
5161-- transform the data shapes of the network.
52−data Network :: [Shape] -> * where
53− O :: (Show x, Layer x i o, KnownShape o, KnownShape i)
54− => !x
55− -> Network '[i, o]
56− (:~>) :: (Show x, Layer x i h, KnownShape h, KnownShape i)
57− => !x
58− -> !(Network (h ': hs))
59− -> Network (i ': h ': hs)
62+data Network :: [*] -> [Shape] -> * where
63+ O :: Layer x i o => !x -> Network '[x] '[i, o]
64+ (:~>) :: Layer x i h => !x -> !(Network xs (h ': hs)) -> Network (x ': xs) (i ': h ': hs)
6065infixr 5 :~>
6166
62−instance Show (Network h) where
67+instance Show (Network l h) where
6368 show (O a) = "O " ++ show a
6469 show (i :~> o) = show i ++ "\n:~>\n" ++ show o
70+
71+-- | Gradients of a network.
72+-- Parameterised on the layers of a Network.
73+data Gradients :: [*] -> * where
74+ OG :: UpdateLayer x => Gradient x -> Gradients '[x]
75+ (:/>) :: UpdateLayer x => Gradient x -> Gradients xs -> Gradients (x ': xs)
76+
77+
78+-- | A network can easily be created by hand with (:~>), but an easy way to initialise a random
79+-- network is with the randomNetwork.
80+class CreatableNetwork (xs :: [*]) (ss :: [Shape]) where
81+ -- | Create a network of the types requested
82+ randomNetwork :: MonadRandom m => m (Network xs ss)
83+
84+instance Layer x i o => CreatableNetwork (x ': '[]) (i ': o ': '[]) where
85+ randomNetwork = O <$> createRandom
86+
87+instance (Layer x i o, CreatableNetwork xs (o ': r ': rs)) => CreatableNetwork (x ': xs) (i ': o ': r ': rs) where
88+ randomNetwork = (:~>) <$> createRandom <*> randomNetwork
src/Grenade/Core/Runner.hs+35−23
@@ -7,52 +7,64 @@
77
88module Grenade.Core.Runner (
99 train
10+ , backPropagate
1011 , runNet
12+ , applyUpdate
1113 ) where
1214
1315import Data.Singletons.Prelude
1416import Grenade.Core.Network
1517import Grenade.Core.Shape
1618
17−-- | Update a network with new weights after training with an instance.
18−train :: forall i o hs. (Head hs ~ i, Last hs ~ o, KnownShape i, KnownShape o)
19− => LearningParameters -- ^ learning rate
20− -> S' i -- ^ input vector
21− -> S' o -- ^ target vector
22− -> Network hs -- ^ network to train
23− -> Network hs
24−train rate x0 target = fst . go x0
19+-- | Drive and network and collect it's back propogated gradients.
20+backPropagate :: forall input output shapes layers. (Head shapes ~ input, Last shapes ~ output)
21+ => Network layers shapes -> S' input -> S' output -> Gradients layers
22+backPropagate network input target =
23+ fst $ go input network
2524 where
26− go :: forall j js. (Head js ~ j, Last js ~ o, KnownShape j)
27− => S' j -- ^ input vector
28− -> Network js -- ^ network to train
29− -> (Network js, S' j)
25+ go :: forall j js sublayers. (Head js ~ j, Last js ~ output)
26+ => S' j -- ^ input vector
27+ -> Network sublayers js -- ^ network to train
28+ -> (Gradients sublayers, S' j)
3029 -- handle input from the beginning, feeding upwards.
3130 go !x (layer :~> n)
3231 = let y = runForwards layer x
33− -- run the rest of the network, and get the layer from above.
32+ -- recursively run the rest of the network, and get the layer from above.
3433 (n', dWs') = go y n
3534 -- calculate the gradient for this layer to pass down,
3635 (layer', dWs) = runBackards layer x dWs'
3736
38− -- Update this layer using the gradient
39− newLayer = runUpdate rate layer layer'
40−
41− in (newLayer :~> n', dWs)
37+ in (layer' :/> n', dWs)
4238
4339 -- handle the output layer, bouncing the derivatives back down.
4440 go !x (O layer)
4541 = let y = runForwards layer x
46− -- the gradient (how much y affects the error)
42+ -- the gradient (how much y affects the error)
4743 (layer', dWs) = runBackards layer x (y - target)
48− newLayer = runUpdate rate layer layer'
4944
50− in (O newLayer, dWs)
45+ in (OG layer', dWs)
46+
47+-- | Update a network with new weights after training with an instance.
48+train :: forall input output shapes layers. (Head shapes ~ input, Last shapes ~ output)
49+ => LearningParameters -- ^ learning rate
50+ -> Network layers shapes -- ^ network to train
51+ -> S' input -> S' output -- ^ target vector
52+ -> Network layers shapes
53+train rate network input output =
54+ let grads = backPropagate network input output
55+ in applyUpdate rate network grads
56+
57+applyUpdate :: LearningParameters -> Network ls ss -> Gradients ls -> Network ls ss
58+applyUpdate rate (O layer) (OG gradient)
59+ = O (runUpdate rate layer gradient)
60+applyUpdate rate (layer :~> rest) (gradient :/> grest)
61+ = runUpdate rate layer gradient :~> applyUpdate rate rest grest
62+applyUpdate _ _ _
63+ = error "Impossible for the gradients of a network to have a different length to the network"
5164
5265-- | Just forwards propagation with no training.
53−runNet :: Network hs
66+runNet :: Network layers hs
5467 -> S' (Head hs) -- ^ input vector
5568 -> S' (Last hs) -- ^ target vector
56−runNet (layer :~> n) !x = let y = runForwards layer x
57− in runNet n y
69+runNet (layer :~> n) !x = let y = runForwards layer x in runNet n y
5870runNet (O layer) !x = runForwards layer x
src/Grenade/Core/Shape.hs+1−18
@@ -1,9 +1,7 @@
1−{-# LANGUAGE BangPatterns #-}
21{-# LANGUAGE DataKinds #-}
32{-# LANGUAGE GADTs #-}
43{-# LANGUAGE KindSignatures #-}
54{-# LANGUAGE ScopedTypeVariables #-}
6−{-# LANGUAGE StandaloneDeriving #-}
75{-# LANGUAGE TypeOperators #-}
86{-# LANGUAGE TypeFamilies #-}
97{-# LANGUAGE PolyKinds #-}
@@ -18,11 +16,9 @@
1816module Grenade.Core.Shape (
1917 Shape (..)
2018 , S' (..)
21− , KnownShape (..)
2219 ) where
2320
2421import Data.Singletons.TypeLits
25−import Data.Proxy
2622
2723import Numeric.LinearAlgebra.Static
2824
@@ -36,7 +32,7 @@ data Shape =
3632 | D2 Nat Nat
3733 | D3 Nat Nat Nat
3834
39−instance KnownShape x => Num (S' x) where
35+instance Num (S' x) where
4036 (+) (S1D' x) (S1D' y) = S1D' (x + y)
4137 (+) (S2D' x) (S2D' y) = S2D' (x + y)
4238 (+) (S3D' x) (S3D' y) = S3D' (vectorZip (+) x y)
@@ -72,16 +68,3 @@ instance Show (S' n) where
7268 show (S1D' a) = "S1D' " ++ show a
7369 show (S2D' a) = "S2D' " ++ show a
7470 show (S3D' a) = "S3D' " ++ show a
75−
76−-- | Singleton for Shape
77−class KnownShape (n :: Shape) where
78− shapeSing :: Proxy n
79−
80−instance KnownShape ('D1 n) where
81− shapeSing = Proxy
82−
83−instance KnownShape ('D2 n m) where
84− shapeSing = Proxy
85−
86−instance KnownShape ('D3 l n m) where
87− shapeSing = Proxy
src/Grenade/Layers/Convolution.hs+12−1
@@ -119,7 +119,14 @@ randomConvolution = do
119119 mm = konst 0
120120 return $ Convolution wN mm
121121
122−instance UpdateLayer (Convolution channels filters kernelRows kernelCols strideRows strideCols) where
122+instance ( KnownNat channels
123+ , KnownNat filters
124+ , KnownNat kernelRows
125+ , KnownNat kernelColumns
126+ , KnownNat strideRows
127+ , KnownNat strideColumns
128+ , KnownNat (kernelRows * kernelColumns * channels)
129+ ) => UpdateLayer (Convolution channels filters kernelRows kernelColumns strideRows strideColumns) where
123130 type Gradient (Convolution channels filters kernelRows kernelCols strideRows strideCols) = (Convolution' channels filters kernelRows kernelCols strideRows strideCols)
124131 runUpdate LearningParameters {..} (Convolution oldKernel oldMomentum) (Convolution' kernelGradient) =
125132 let newMomentum = konst learningMomentum * oldMomentum - konst learningRate * kernelGradient
@@ -127,6 +134,8 @@ instance UpdateLayer (Convolution channels filters kernelRows kernelCols strideR
127134 newKernel = oldKernel + newMomentum - regulariser
128135 in Convolution newKernel newMomentum
129136
137+ createRandom = randomConvolution
138+
130139-- | A two dimentional image may have a convolution filter applied to it
131140instance ( KnownNat kernelRows
132141 , KnownNat kernelCols
@@ -139,6 +148,7 @@ instance ( KnownNat kernelRows
139148 , KnownNat outputCols
140149 , ((outputRows - 1) * strideRows) ~ (inputRows - kernelRows)
141150 , ((outputCols - 1) * strideCols) ~ (inputCols - kernelCols)
151+ , KnownNat (kernelRows * kernelCols * 1)
142152 ) => Layer (Convolution 1 filters kernelRows kernelCols strideRows strideCols) ('D2 inputRows inputCols) ('D3 outputRows outputCols filters) where
143153 runForwards (Convolution kernel _) (S2D' input) =
144154 let ex = extract input
@@ -192,6 +202,7 @@ instance ( KnownNat kernelRows
192202 , KnownNat channels
193203 , ((outputRows - 1) * strideRows) ~ (inputRows - kernelRows)
194204 , ((outputCols - 1) * strideCols) ~ (inputCols - kernelCols)
205+ , KnownNat (kernelRows * kernelCols * channels)
195206 ) => Layer (Convolution channels filters kernelRows kernelCols strideRows strideCols) ('D3 inputRows inputCols channels) ('D3 outputRows outputCols filters) where
196207 runForwards (Convolution kernel _) (S3D' input) =
197208 let ex = vecToList $ fmap extract input
src/Grenade/Layers/Crop.hs+2−5
@@ -28,11 +28,7 @@ data Crop :: Nat
2828 -> Nat
2929 -> Nat
3030 -> Nat -> * where
31− Crop :: ( KnownNat cropLeft
32− , KnownNat cropTop
33− , KnownNat cropRight
34− , KnownNat cropBottom
35− ) => Crop cropLeft cropTop cropRight cropBottom
31+ Crop :: Crop cropLeft cropTop cropRight cropBottom
3632
3733instance Show (Crop cropLeft cropTop cropRight cropBottom) where
3834 show Crop = "Crop"
@@ -40,6 +36,7 @@ instance Show (Crop cropLeft cropTop cropRight cropBottom) where
4036instance UpdateLayer (Crop l t r b) where
4137 type Gradient (Crop l t r b) = ()
4238 runUpdate _ x _ = x
39+ createRandom = return Crop
4340
4441-- | A two dimentional image can be cropped.
4542instance ( KnownNat cropLeft
src/Grenade/Layers/Dropout.hs+1
@@ -34,6 +34,7 @@ data Dropout o =
3434instance (KnownNat i) => UpdateLayer (Dropout i) where
3535 type Gradient (Dropout i) = ()
3636 runUpdate _ x _ = x
37+ createRandom = randomDropout 0.95
3738
3839randomDropout :: (MonadRandom m, KnownNat i)
3940 => Double -> m (Dropout i)
src/Grenade/Layers/Flatten.hs+2
@@ -28,6 +28,8 @@ data FlattenLayer = FlattenLayer
2828instance UpdateLayer FlattenLayer where
2929 type Gradient FlattenLayer = ()
3030 runUpdate _ _ _ = FlattenLayer
31+ createRandom = return FlattenLayer
32+
3133
3234instance (KnownNat a, KnownNat x, KnownNat y, a ~ (x * y)) => Layer FlattenLayer ('D2 x y) ('D1 a) where
3335 runForwards _ (S2D' y) = S1D' . fromList . toList . flatten . extract $ y
src/Grenade/Layers/FullyConnected.hs+2
@@ -45,6 +45,8 @@ instance (KnownNat i, KnownNat o) => UpdateLayer (FullyConnected i o) where
4545 newActivations = oldActivations + newMomentum - regulariser
4646 in FullyConnected newBias newBiasMomentum newActivations newMomentum
4747
48+ createRandom = randomFullyConnected
49+
4850instance (KnownNat i, KnownNat o) => Layer (FullyConnected i o) ('D1 i) ('D1 o) where
4951 -- Do a matrix vector multiplication and return the result.
5052 runForwards (FullyConnected wB _ wN _) (S1D' v) = S1D' (wB + wN #> v)
src/Grenade/Layers/Fuse.hs+5−4
@@ -23,23 +23,24 @@ import Grenade.Core.Shape
2323-- This does however have a trade off, internal incremental states in the Wengert tape are
2424-- not retained during reverse accumulation. So less RAM is used, but more compute is required.
2525data Fuse :: * -> * -> Shape -> Shape -> Shape -> * where
26− (:$$) :: (Show x, Show y, Layer x i h, Layer y h o, KnownShape h, KnownShape i, KnownShape o)
26+ (:$$) :: (Layer x i h, Layer y h o)
2727 => !x
2828 -> !y
2929 -> Fuse x y i h o
3030infixr 5 :$$
3131
32−instance Show (Fuse x y i h o) where
32+instance (Show x, Show y) => Show (Fuse x y i h o) where
3333 show (x :$$ y) = "(" ++ show x ++ " :$$ " ++ show y ++ ")"
3434
35−instance (KnownShape i, KnownShape h, KnownShape o) => UpdateLayer (Fuse x y i h o) where
35+instance (Layer x i h, Layer y h o) => UpdateLayer (Fuse x y i h o) where
3636 type Gradient (Fuse x y i h o) = (Gradient x, Gradient y)
3737 runUpdate lr (x :$$ y) (x', y') =
3838 let newX = runUpdate lr x x'
3939 newY = runUpdate lr y y'
4040 in (newX :$$ newY)
41+ createRandom = (:$$) <$> createRandom <*> createRandom
4142
42−instance (KnownShape i, KnownShape h, KnownShape o) => Layer (Fuse x y i h o) i o where
43+instance (Layer x i h, Layer y h o) => Layer (Fuse x y i h o) i o where
4344 runForwards (x :$$ y) input =
4445 let yInput :: S' h = runForwards x input
4546 in runForwards y yInput
src/Grenade/Layers/Logit.hs+1
@@ -25,6 +25,7 @@ data Logit = Logit
2525instance UpdateLayer Logit where
2626 type Gradient Logit = ()
2727 runUpdate _ _ _ = Logit
28+ createRandom = return Logit
2829
2930instance (KnownNat i) => Layer Logit ('D1 i) ('D1 i) where
3031 runForwards _ (S1D' y) = S1D' (logistic y)
src/Grenade/Layers/Pad.hs+2−5
@@ -28,11 +28,7 @@ data Pad :: Nat
2828 -> Nat
2929 -> Nat
3030 -> Nat -> * where
31− Pad :: ( KnownNat padLeft
32− , KnownNat padTop
33− , KnownNat padRight
34− , KnownNat padBottom
35− ) => Pad padLeft padTop padRight padBottom
31+ Pad :: Pad padLeft padTop padRight padBottom
3632
3733instance Show (Pad padLeft padTop padRight padBottom) where
3834 show Pad = "Pad"
@@ -40,6 +36,7 @@ instance Show (Pad padLeft padTop padRight padBottom) where
4036instance UpdateLayer (Pad l t r b) where
4137 type Gradient (Pad l t r b) = ()
4238 runUpdate _ x _ = x
39+ createRandom = return Pad
4340
4441-- | A two dimentional image can be padped.
4542instance ( KnownNat padLeft
src/Grenade/Layers/Pooling.hs+2−5
@@ -41,11 +41,7 @@ data Pooling :: Nat
4141 -> Nat
4242 -> Nat
4343 -> Nat -> * where
44− Pooling :: ( KnownNat kernelRows
45− , KnownNat kernelColumns
46− , KnownNat strideRows
47− , KnownNat strideColumns
48− ) => Pooling kernelRows kernelColumns strideRows strideColumns
44+ Pooling :: Pooling kernelRows kernelColumns strideRows strideColumns
4945
5046instance Show (Pooling k k' s s') where
5147 show Pooling = "Pooling"
@@ -54,6 +50,7 @@ instance Show (Pooling k k' s s') where
5450instance UpdateLayer (Pooling kernelRows kernelColumns strideRows strideColumns) where
5551 type Gradient (Pooling kr kc sr sc) = ()
5652 runUpdate _ Pooling _ = Pooling
53+ createRandom = return Pooling
5754
5855-- | A two dimentional image can be pooled.
5956instance ( KnownNat kernelRows
src/Grenade/Layers/Relu.hs+1
@@ -25,6 +25,7 @@ data Relu = Relu
2525instance UpdateLayer Relu where
2626 type Gradient Relu = ()
2727 runUpdate _ _ _ = Relu
28+ createRandom = return Relu
2829
2930instance ( KnownNat i) => Layer Relu ('D1 i) ('D1 i) where
3031 runForwards _ (S1D' y) = S1D' (relu y)
src/Grenade/Layers/Tanh.hs+2−1
@@ -15,13 +15,14 @@ import Grenade.Core.Network
1515import Grenade.Core.Shape
1616
1717-- | A Tanh layer.
18−-- A layer which can act between any shape of the same dimension, perfoming an tanh function.s
18+-- A layer which can act between any shape of the same dimension, perfoming a tanh function.
1919data Tanh = Tanh
2020 deriving Show
2121
2222instance UpdateLayer Tanh where
2323 type Gradient Tanh = ()
2424 runUpdate _ _ _ = Tanh
25+ createRandom = return Tanh
2526
2627instance KnownNat i => Layer Tanh ('D1 i) ('D1 i) where
2728 runForwards _ (S1D' y) = S1D' (tanh y)