e710046 · grenade · git.iko.soy

git.iko.soy/grenade

on master help sign in

Implement Dropout layer

e710046Huw Campbell10 years ago

src/Grenade/Core/Network.hs+1−1

@@ -33,7 +33,7 @@ class Layer (m :: * -> *) x (i :: Shape) (o :: Shape) where

33-- | Type of a network.

34-- The [Shape] type specifies the shapes of data passed between the layers.

35-- Could be considered to be a heterogeneous list of layers which are able to

36−-- transform the date shapes of the network.

36+-- transform the data shapes of the network.

37data Network :: (* -> *) -> [Shape] -> * where

38 O :: (Show x, Layer m x i o, KnownShape o, KnownShape i)

39 => !x

src/Grenade/Layers/Dropout.hs+13−2

@@ -19,6 +19,7 @@ import Grenade.Core.Network

20import Numeric.LinearAlgebra.Static

22+

22-- Dropout layer help to reduce overfitting.

23-- Dropout layer help to reduce overfitting.

23-- Idea here is that the vector is a shape of 1s and 0s, which we multiply the input by.

24-- Idea here is that the vector is a shape of 1s and 0s, which we multiply the input by.

24-- After backpropogation, we return a new matrix/vector, with different bits dropped out.

25-- After backpropogation, we return a new matrix/vector, with different bits dropped out.

@@ -27,6 +28,16 @@ import Numeric.LinearAlgebra.Static

27data Dropout o = Dropout Double (R o)

28data Dropout o = Dropout Double (R o)

28 deriving Show

29 deriving Show

31+randomDropout :: (MonadRandom m, KnownNat i)

32+ => Double -> m (Dropout i)

33+randomDropout rate = do

34+ seed <- getRandom

35+ let wN = randomVector seed Uniform

36+ xs = dvmap (\a -> if a <= rate then 0 else 1) wN

37+ return $ Dropout rate xs

38+

30instance (MonadRandom m, KnownNat i) => Layer m (Dropout i) ('D1 i) ('D1 i) where

39instance (MonadRandom m, KnownNat i) => Layer m (Dropout i) ('D1 i) ('D1 i) where

31− runForwards _ _= error "todo"

40+ runForwards (Dropout _ drops) (S1D' x) = return . S1D' $ x * drops

32− runBackards _ _ _ _ = error "todo"

41+ runBackards _ (Dropout rate drops) _ (S1D' x) = do

42+ newDropout <- randomDropout rate

43+ return (newDropout, S1D' $ x * drops)