e710046 · grenade · git.iko.soy

git.iko.soy/grenade

Implement Dropout layer

commite710046a3bca01a5d5172147d4ab0392ceb74176 authorHuw Campbell <huw.campbell@ambiata.com> dateMon 27 Jun 2016, 17:59 (10 years ago) parent3d1919f

2 files changed+14−3

unifiedsplit

src/Grenade/Core/Network.hs+1−1

@@ -33,7 +33,7 @@ class Layer (m :: * -> *) x (i :: Shape) (o :: Shape) where

3333-- | Type of a network.

3434-- The [Shape] type specifies the shapes of data passed between the layers.

3535-- Could be considered to be a heterogeneous list of layers which are able to

36−-- transform the date shapes of the network.

36+-- transform the data shapes of the network.

3737data Network :: (* -> *) -> [Shape] -> * where

3838 O :: (Show x, Layer m x i o, KnownShape o, KnownShape i)

3939 => !x

src/Grenade/Layers/Dropout.hs+13−2

@@ -19,6 +19,7 @@ import Grenade.Core.Network

1919

2020import Numeric.LinearAlgebra.Static

2121

22+

2223-- Dropout layer help to reduce overfitting.

2324-- Idea here is that the vector is a shape of 1s and 0s, which we multiply the input by.

2425-- After backpropogation, we return a new matrix/vector, with different bits dropped out.

@@ -27,6 +28,16 @@ import Numeric.LinearAlgebra.Static

2728data Dropout o = Dropout Double (R o)

2829 deriving Show

2930

31+randomDropout :: (MonadRandom m, KnownNat i)

32+ => Double -> m (Dropout i)

33+randomDropout rate = do

34+ seed <- getRandom

35+ let wN = randomVector seed Uniform

36+ xs = dvmap (\a -> if a <= rate then 0 else 1) wN

37+ return $ Dropout rate xs

38+

3039instance (MonadRandom m, KnownNat i) => Layer m (Dropout i) ('D1 i) ('D1 i) where

31− runForwards _ _= error "todo"

32− runBackards _ _ _ _ = error "todo"

40+ runForwards (Dropout _ drops) (S1D' x) = return . S1D' $ x * drops

41+ runBackards _ (Dropout rate drops) _ (S1D' x) = do

42+ newDropout <- randomDropout rate

43+ return (newDropout, S1D' $ x * drops)