ilyakooo0/grenade
1
0
Fork 0
mirror of https://github.com/ilyakooo0/grenade.git synced 2024-10-26 14:32:47 +03:00
Code Issues Projects Releases Wiki Activity

Separate Gradient calculation from update

Browse Source
This commit is contained in:
Huw Campbell 2016-12-02 18:22:35 +11:00
parent cef1b14268
commit 114dab4103
17 changed files with 193 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
grenade.cabal
View File

@ -14,6 +14,7 @@ library
build-depends:
base >= 4.8 && < 5
, bytestring == 0.10.*
, async
, either == 4.4.*
, exceptions == 0.8.*
, hmatrix

18
main/feedforward.hs
View File

@ -35,14 +35,13 @@ randomNet = do
c :: FullyConnected 10 1 <- randomFullyConnected
return $ a :~> Tanh :~> b :~> Relu :~> c :~> O Logit
netTest :: MonadRandom m => Double -> Int -> m String
netTest :: MonadRandom m => LearningParameters -> Int -> m String
netTest rate n = do
inps <- replicateM n $ do
s <- getRandom
s <- getRandom
return $ S1D' $ SA.randomVector s SA.Uniform * 2 - 1
let outs = flip map inps $ \(S1D' v) ->
if v `inCircle` (fromRational 0.33, 0.33)
|| v `inCircle` (fromRational (-0.33), 0.33)
if v `inCircle` (fromRational 0.33, 0.33) || v `inCircle` (fromRational (-0.33), 0.33)
then S1D' $ fromRational 1
else S1D' $ fromRational 0
net0 <- randomNet
@ -70,11 +69,16 @@ netTest rate n = do
normx (S1D' r) = SA.mean r
data FeedForwardOpts = FeedForwardOpts Int Double
data FeedForwardOpts = FeedForwardOpts Int LearningParameters
feedForward' :: Parser FeedForwardOpts
feedForward' = FeedForwardOpts <$> option auto (long "examples" <> short 'e' <> value 1000000)
<*> option auto (long "train_rate" <> short 'r' <> value 0.01)
feedForward' =
FeedForwardOpts <$> option auto (long "examples" <> short 'e' <> value 1000000)
<*> (LearningParameters
<$> option auto (long "train_rate" <> short 'r' <> value 0.01)
<*> option auto (long "momentum" <> value 0.9)
<*> option auto (long "l2" <> value 0.0001)
)
main :: IO ()
main = do

16
main/mnist.hs
View File

@ -43,7 +43,7 @@ randomMnistNet = do
f :: FullyConnected 80 10 <- randomFullyConnected
return $ pad :~> a :~> b :~> Relu :~> c :~> d :~> FlattenLayer :~> Relu :~> e :~> Logit :~> f :~> O Logit
convTest :: Int -> FilePath -> FilePath -> Double -> IO ()
convTest :: Int -> FilePath -> FilePath -> LearningParameters -> IO ()
convTest iterations trainFile validateFile rate = do
net0 <- evalRandIO randomMnistNet
fT <- T.readFile trainFile
@ -52,7 +52,7 @@ convTest iterations trainFile validateFile rate = do
let validateRows = traverse (A.parseOnly p) (T.lines fV)
case (trainRows, validateRows) of
(Right tr', Right vr') -> foldM_ (runIteration tr' vr') net0 [1..iterations]
err -> putStrLn $ show err
err -> print err
where
trainEach !rate' !nt !(i, o) = train rate' i o nt
@ -65,20 +65,24 @@ convTest iterations trainFile validateFile rate = do
return (S2D' $ SA.fromList pixels, S1D' $ SA.fromList lab')
runIteration trainRows validateRows net i = do
let trained' = runIdentity $ foldM (trainEach (rate * (0.9 ^ i))) net trainRows
let trained' = runIdentity $ foldM (trainEach rate) net trainRows
let res = runIdentity $ traverse (\(rowP,rowL) -> (rowL,) <$> runNet trained' rowP) validateRows
let res' = fmap (\(S1D' label, S1D' prediction) -> (maxIndex (SA.extract label), maxIndex (SA.extract prediction))) res
putStrLn $ show trained'
print trained'
putStrLn $ "Iteration " ++ show i ++ ": " ++ show (length (filter ((==) <$> fst <*> snd) res')) ++ " of " ++ show (length res')
return trained'
data MnistOpts = MnistOpts FilePath FilePath Int Double
data MnistOpts = MnistOpts FilePath FilePath Int LearningParameters
mnist' :: Parser MnistOpts
mnist' = MnistOpts <$> (argument str (metavar "TRAIN"))
<*> (argument str (metavar "VALIDATE"))
<*> option auto (long "iterations" <> short 'i' <> value 15)
<*> option auto (long "train_rate" <> short 'r' <> value 0.01)
<*> (LearningParameters
<$> option auto (long "train_rate" <> short 'r' <> value 0.01)
<*> option auto (long "momentum" <> value 0.9)
<*> option auto (long "l2" <> value 0.0001)
)
main :: IO ()
main = do

13
src/Grenade.hs
View File

@ -1,16 +1,3 @@
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE GADTs #-}
{-# LANGUAGE KindSignatures #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE PolyKinds #-}
{-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleInstances #-}
module Grenade (
module X
) where

36
src/Grenade/Core/Network.hs
View File

@ -1,9 +1,7 @@
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE GADTs #-}
{-# LANGUAGE KindSignatures #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE PolyKinds #-}
@ -14,31 +12,49 @@
module Grenade.Core.Network (
Layer (..)
, Network (..)
, UpdateLayer (..)
, LearningParameters (..)
) where
import Data.Typeable
import Grenade.Core.Shape
data LearningParameters = LearningParameters {
learningRate :: Double
, learningMomentum :: Double
, learningRegulariser :: Double
} deriving (Eq, Show)
-- | Class for updating a layer. All layers implement this, and it is
-- shape independent.
class UpdateLayer (m :: * -> *) x where
-- | The type for the gradient for this layer.
-- Unit if there isn't a gradient to pass back.
type Gradient x :: *
-- | Update a layer with its gradient and learning parameters
runUpdate :: LearningParameters -> x -> Gradient x -> m x
-- | Class for a layer. All layers implement this, however, they don't
-- need to implement it for all shapes, only ones which are appropriate.
class Layer (m :: * -> *) x (i :: Shape) (o :: Shape) where
class UpdateLayer m x => Layer (m :: * -> *) x (i :: Shape) (o :: Shape) where
-- | Used in training and scoring. Take the input from the previous
-- layer, and give the output from this layer.
runForwards :: x -> S' i -> m (S' o)
-- | Back propagate a step. Takes a learning rate (move from here?)
-- the current layer, the input that the layer gave from the input
-- and the back propagated derivatives from the layer above.
-- Returns the updated layer and the derivatives to push back further.
runBackards :: Double -> x -> S' i -> S' o -> m (x, S' i)
-- | Back propagate a step. Takes the current layer, the input that the
-- layer gave from the input and the back propagated derivatives from
-- the layer above.
-- Returns the gradient layer and the derivatives to push back further.
runBackards :: x -> S' i -> S' o -> m (Gradient x, S' i)
-- | Type of a network.
-- The [Shape] type specifies the shapes of data passed between the layers.
-- Could be considered to be a heterogeneous list of layers which are able to
-- transform the data shapes of the network.
data Network :: (* -> *) -> [Shape] -> * where
O :: (Show x, Layer m x i o, KnownShape o, KnownShape i)
O :: (Typeable x, Show x, Layer m x i o, KnownShape o, KnownShape i)
=> !x
-> Network m '[i, o]
(:~>) :: (Show x, Layer m x i h, KnownShape h, KnownShape i)
(:~>) :: (Typeable x, Show x, Layer m x i h, KnownShape h, KnownShape i)
=> !x
-> !(Network m (h ': hs))
-> Network m (i ': h ': hs)

22
src/Grenade/Core/Runner.hs
View File

@ -1,9 +1,7 @@
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE GADTs #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE KindSignatures #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
@ -18,7 +16,7 @@ import Grenade.Core.Shape
-- | Update a network with new weights after training with an instance.
train :: forall m i o hs. (Monad m, Head hs ~ i, Last hs ~ o, KnownShape i, KnownShape o)
=> Double -- ^ learning rate
=> LearningParameters -- ^ learning rate
-> S' i -- ^ input vector
-> S' o -- ^ target vector
-> Network m hs -- ^ network to train
@ -35,20 +33,26 @@ train rate x0 target = fmap fst . go x0
-- run the rest of the network, and get the layer from above.
(n', dWs') <- go y n
-- calculate the gradient for this layer to pass down,
(layer', dWs) <- runBackards rate layer x dWs'
return (layer' :~> n', dWs)
(layer', dWs) <- runBackards layer x dWs'
-- Update this layer using the gradient
newLayer <- runUpdate rate layer layer'
return (newLayer :~> n', dWs)
-- handle the output layer, bouncing the derivatives back down.
go !x (O layer)
= do y <- runForwards layer x
= do y <- runForwards layer x
-- the gradient (how much y affects the error)
(layer', dWs) <- runBackards rate layer x (y - target)
return (O layer', dWs)
(layer', dWs) <- runBackards layer x (y - target)
newLayer <- runUpdate rate layer layer'
return (O newLayer, dWs)
-- | Just forwards propagation with no training.
runNet :: forall m hs. (Monad m)
=> Network m hs
-> (S' (Head hs)) -- ^ input vector
-> S' (Head hs) -- ^ input vector
-> m (S' (Last hs)) -- ^ target vector
runNet (layer :~> n) !x = do y <- runForwards layer x
runNet n y

50
src/Grenade/Layers/Convolution.hs
View File

@ -2,6 +2,7 @@
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE RecordWildCards #-}
{-# LANGUAGE GADTs #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
@ -65,6 +66,24 @@ data Convolution :: Nat -- ^ Number of channels, for the first layer this could
-> !(L kernelFlattened filters) -- ^ The last kernel update (or momentum)
-> Convolution channels filters kernelRows kernelColumns strideRows strideColumns
data Convolution' :: Nat -- ^ Number of channels, for the first layer this could be RGB for instance.
-> Nat -- ^ Number of filters, this is the number of channels output by the layer.
-> Nat -- ^ The number of rows in the kernel filter
-> Nat -- ^ The number of column in the kernel filter
-> Nat -- ^ The row stride of the convolution filter
-> Nat -- ^ The columns stride of the convolution filter
-> * where
Convolution' :: ( KnownNat channels
, KnownNat filters
, KnownNat kernelRows
, KnownNat kernelColumns
, KnownNat strideRows
, KnownNat strideColumns
, KnownNat kernelFlattened
, kernelFlattened ~ (kernelRows * kernelColumns * channels))
=> !(L kernelFlattened filters) -- ^ The kernel filter gradient
-> Convolution' channels filters kernelRows kernelColumns strideRows strideColumns
instance Show (Convolution c f k k' s s') where
show (Convolution a _) = renderConv a
where
@ -99,6 +118,22 @@ randomConvolution = do
mm = konst 0
return $ Convolution wN mm
instance ( Monad m
, KnownNat kernelRows
, KnownNat kernelCols
, KnownNat channels
, KnownNat filters
, KnownNat strideRows
, KnownNat strideCols
, kernelFlattened ~ (kernelRows * kernelColumns * channels)
) => UpdateLayer m (Convolution channels filters kernelRows kernelCols strideRows strideCols) where
type Gradient (Convolution channels filters kernelRows kernelCols strideRows strideCols) = (Convolution' channels filters kernelRows kernelCols strideRows strideCols)
runUpdate LearningParameters {..} (Convolution oldKernel oldMomentum) (Convolution' kernelGradient) = do
let newMomentum = konst learningMomentum * oldMomentum - konst learningRate * kernelGradient
regulariser = konst (learningRegulariser * learningRate) * oldKernel
newKernel = oldKernel + newMomentum - regulariser
return $ Convolution newKernel newMomentum
-- | A two dimentional image may have a convolution filter applied to it
instance ( Monad m
, KnownNat kernelRows
@ -127,7 +162,7 @@ instance ( Monad m
r = col2vid 1 1 1 1 ox oy mt
rs = fmap (fromJust . create) r
in return . S3D' $ mkVector rs
runBackards rate (Convolution kernel momentum) (S2D' input) (S3D' dEdy) =
runBackards (Convolution kernel _) (S2D' input) (S3D' dEdy) =
let ex = extract input
ix = fromIntegral $ natVal (Proxy :: Proxy inputRows)
iy = fromIntegral $ natVal (Proxy :: Proxy inputCols)
@ -145,14 +180,10 @@ instance ( Monad m
vs = vid2col 1 1 1 1 ox oy eo
kN = fromJust . create $ tr c LA.<> vs
mm = momentum * 0.9 - konst rate * kN
wd = konst (0.0001 * rate) * kernel
rM = kernel + mm - wd
dW = vs LA.<> tr ek
xW = col2im kx ky sx sy ix iy dW
in return (Convolution rM mm, S2D' . fromJust . create $ xW)
in return (Convolution' kN, S2D' . fromJust . create $ xW)
-- | A three dimensional image (or 2d with many channels) can have
@ -187,7 +218,7 @@ instance ( Monad m
r = col2vid 1 1 1 1 ox oy mt
rs = fmap (fromJust . create) r
in return . S3D' $ mkVector rs
runBackards rate (Convolution kernel momentum) (S3D' input) (S3D' dEdy) =
runBackards (Convolution kernel _) (S3D' input) (S3D' dEdy) =
let ex = vecToList $ fmap extract input
ix = fromIntegral $ natVal (Proxy :: Proxy inputRows)
iy = fromIntegral $ natVal (Proxy :: Proxy inputCols)
@ -205,14 +236,11 @@ instance ( Monad m
vs = vid2col 1 1 1 1 ox oy eo
kN = fromJust . create $ tr c LA.<> vs
mm = momentum * 0.9 - konst rate * kN
wd = konst (0.0005 * rate) * kernel
rM = kernel + mm - wd
dW = vs LA.<> tr ek
xW = col2vid kx ky sx sy ix iy dW
in return (Convolution rM mm, S3D' . mkVector . fmap (fromJust . create) $ xW)
in return (Convolution' kN, S3D' . mkVector . fmap (fromJust . create) $ xW)
im2col :: Int -> Int -> Int -> Int -> Matrix Double -> Matrix Double
im2col nrows ncols srows scols m =

10
src/Grenade/Layers/Crop.hs
View File

@ -1,7 +1,5 @@
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE GADTs #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
@ -39,6 +37,10 @@ data Crop :: Nat
instance Show (Crop cropLeft cropTop cropRight cropBottom) where
show Crop = "Crop"
instance Monad m => UpdateLayer m (Crop l t r b) where
type Gradient (Crop l t r b) = ()
runUpdate _ x _ = return x
-- | A two dimentional image can be cropped.
instance ( Monad m
, KnownNat cropLeft
@ -60,11 +62,11 @@ instance ( Monad m
m = extract input
r = subMatrix (cropt, cropl) (nrows, ncols) m
in return . S2D' . fromJust . create $ r
runBackards _ crop _ (S2D' dEdy) =
runBackards _ _ (S2D' dEdy) =
let cropl = fromIntegral $ natVal (Proxy :: Proxy cropLeft)
cropt = fromIntegral $ natVal (Proxy :: Proxy cropTop)
cropr = fromIntegral $ natVal (Proxy :: Proxy cropRight)
cropb = fromIntegral $ natVal (Proxy :: Proxy cropBottom)
eo = extract dEdy
vs = diagBlock [konst 0 (cropt,cropl), eo, konst 0 (cropb,cropr)]
in return (crop, S2D' . fromJust . create $ vs)
in return ((), S2D' . fromJust . create $ vs)

14
src/Grenade/Layers/Dropout.hs
View File

@ -1,7 +1,5 @@
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE MultiParamTypeClasses #-}
@ -32,6 +30,10 @@ import Numeric.LinearAlgebra.Static
data Dropout o = Dropout Double (R o)
deriving Show
instance (MonadRandom m, KnownNat i) => UpdateLayer m (Dropout i) where
type Gradient (Dropout i) = ()
runUpdate _ (Dropout rate _) _ = randomDropout rate
randomDropout :: (MonadRandom m, KnownNat i)
=> Double -> m (Dropout i)
randomDropout rate = do
@ -44,8 +46,6 @@ instance (MonadRandom m, MonadState Phase m, KnownNat i) => Layer m (Dropout i)
runForwards (Dropout rate drops) (S1D' x) = isTrainingPhase >>= \case
True -> return . S1D' $ x * drops
False -> return . S1D' $ dvmap (* (1 - rate)) x
runBackards _ oldDropout@(Dropout rate drops) _ (S1D' x) = isTrainingPhase >>= \case
True -> do
newDropout <- randomDropout rate
return (newDropout, S1D' $ x * drops)
False -> return (oldDropout, S1D' $ dvmap (* (1 - rate)) x)
runBackards (Dropout rate drops) _ (S1D' x) = isTrainingPhase >>= \case
True -> return ((), S1D' $ x * drops)
False -> return ((), S1D' $ dvmap (* (1 - rate)) x)

12
src/Grenade/Layers/Flatten.hs
View File

@ -25,20 +25,24 @@ import Grenade.Core.Network
data FlattenLayer = FlattenLayer
deriving Show
instance Monad m => UpdateLayer m FlattenLayer where
type Gradient FlattenLayer = ()
runUpdate _ _ _ = return FlattenLayer
instance (Monad m, KnownNat a, KnownNat x, KnownNat y, a ~ (x * y)) => Layer m FlattenLayer ('D2 x y) ('D1 a) where
runForwards _ (S2D' y) = return $ S1D' . fromList . toList . flatten . extract $ y
runBackards _ _ _ (S1D' y) = return (FlattenLayer, S2D' . fromList . toList . unwrap $ y)
runForwards _ (S2D' y) = return $ S1D' . fromList . toList . flatten . extract $ y
runBackards _ _ (S1D' y) = return ((), S2D' . fromList . toList . unwrap $ y)
instance (Monad m, KnownNat a, KnownNat x, KnownNat y, KnownNat z, a ~ (x * y * z)) => Layer m FlattenLayer ('D3 x y z) ('D1 a) where
runForwards _ (S3D' y) = return $ S1D' . raiseShapeError . create . vjoin . vecToList . fmap (flatten . extract) $ y
runBackards _ _ _ (S1D' o) = do
runBackards _ _ (S1D' o) = do
let x' = fromIntegral $ natVal (Proxy :: Proxy x)
y' = fromIntegral $ natVal (Proxy :: Proxy y)
z' = fromIntegral $ natVal (Proxy :: Proxy z)
vecs = takesV (replicate z' (x' * y')) (extract o)
ls = fmap (raiseShapeError . create . reshape y') vecs
ls' = mkVector ls :: Vector z (L x y)
return (FlattenLayer, S3D' ls')
return ((), S3D' ls')
raiseShapeError :: Maybe a -> a
raiseShapeError (Just x) = x

32
src/Grenade/Layers/FullyConnected.hs
View File

@ -1,7 +1,6 @@
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE RecordWildCards #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE MultiParamTypeClasses #-}
@ -25,25 +24,36 @@ import Grenade.Core.Shape
data FullyConnected i o = FullyConnected
!(R o) -- Bias neuron weights
!(L o i) -- Activation weights
!(L o i) -- Activation momentums
!(L o i) -- Momentum
data FullyConnected' i o = FullyConnected'
!(R o) -- Bias neuron gradient
!(L o i) -- Activation gradient
instance Show (FullyConnected i o) where
show (FullyConnected _ _ _) = "FullyConnected"
show FullyConnected {} = "FullyConnected"
instance (Monad m, KnownNat i, KnownNat o) => UpdateLayer m (FullyConnected i o) where
type Gradient (FullyConnected i o) = (FullyConnected' i o)
runUpdate LearningParameters {..} (FullyConnected oldBias oldActivations oldMomentum) (FullyConnected' biasGradient activationGradient) = do
let newBias = oldBias - konst learningRate * biasGradient
newMomentum = konst learningMomentum * oldMomentum - konst learningRate * activationGradient
regulariser = konst (learningRegulariser * learningRate) * oldActivations
newActivations = oldActivations + newMomentum - regulariser
return $ FullyConnected newBias newActivations newMomentum
instance (Monad m, KnownNat i, KnownNat o) => Layer m (FullyConnected i o) ('D1 i) ('D1 o) where
-- Do a matrix vector multiplication and return the result.
runForwards (FullyConnected wB wN _) (S1D' v) = return $ S1D' (wB + wN #> v)
-- Run a backpropogation step for a full connected layer.
runBackards rate (FullyConnected wB wN mm) (S1D' x) (S1D' dEdy) =
let wB' = wB - konst rate * dEdy
mm' = 0.9 * mm - konst rate * (dEdy `outer` x)
wd' = konst (0.0001 * rate) * wN
wN' = wN + mm' - wd'
w' = FullyConnected wB' wN' mm'
runBackards (FullyConnected _ wN _) (S1D' x) (S1D' dEdy) =
let wB' = dEdy
mm' = dEdy `outer` x
-- calcluate derivatives for next step
dWs = tr wN #> dEdy
in return (w', S1D' dWs)
in return (FullyConnected' wB' mm', S1D' dWs)
randomFullyConnected :: (MonadRandom m, KnownNat i, KnownNat o)
=> m (FullyConnected i o)

24
src/Grenade/Layers/Fuse.hs
View File

@ -3,7 +3,6 @@
{-# LANGUAGE GADTs #-}
{-# LANGUAGE KindSignatures #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE PolyKinds #-}
@ -23,23 +22,30 @@ import Grenade.Core.Shape
-- This can be used to simplify a network if a complicated repeated structure is used.
-- This does however have a trade off, internal incremental states in the Wengert tape are
-- not retained during reverse accumulation. So less RAM is used, but more compute is required.
data Fuse :: (* -> *) -> Shape -> Shape -> Shape -> * where
data Fuse :: (* -> *) -> * -> * -> Shape -> Shape -> Shape -> * where
(:$$) :: (Show x, Show y, Layer m x i h, Layer m y h o, KnownShape h, KnownShape i, KnownShape o)
=> !x
-> !y
-> Fuse m i h o
-> Fuse m x y i h o
infixr 5 :$$
instance Show (Fuse m i h o) where
instance Show (Fuse m x y i h o) where
show (x :$$ y) = "(" ++ show x ++ " :$$ " ++ show y ++ ")"
instance (Monad m, KnownShape i, KnownShape h, KnownShape o) => Layer m (Fuse m i h o) i o where
instance (Monad m, KnownShape i, KnownShape h, KnownShape o) => UpdateLayer m (Fuse m x y i h o) where
type Gradient (Fuse m x y i h o) = (Gradient x, Gradient y)
runUpdate lr (x :$$ y) (x', y') = do
newX <- runUpdate lr x x'
newY <- runUpdate lr y y'
return (newX :$$ newY)
instance (Monad m, KnownShape i, KnownShape h, KnownShape o) => Layer m (Fuse m x y i h o) i o where
runForwards (x :$$ y) input = do
yInput :: S' h <- runForwards x input
runForwards y yInput
runBackards rate (x :$$ y) input backGradient = do
runBackards (x :$$ y) input backGradient = do
yInput :: S' h <- runForwards x input
(y', yGrad) <- runBackards rate y yInput backGradient
(x', xGrad) <- runBackards rate x input yGrad
return (x' :$$ y', xGrad)
(y', yGrad) <- runBackards y yInput backGradient
(x', xGrad) <- runBackards x input yGrad
return ((x', y'), xGrad)

13
src/Grenade/Layers/Logit.hs
View File

@ -1,7 +1,5 @@
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE MultiParamTypeClasses #-}
@ -11,6 +9,7 @@ module Grenade.Layers.Logit (
Logit (..)
) where
import Data.Singletons.TypeLits
import Grenade.Core.Network
import Grenade.Core.Vector
@ -23,17 +22,21 @@ import Grenade.Core.Shape
data Logit = Logit
deriving Show
instance Monad m => UpdateLayer m Logit where
type Gradient Logit = ()
runUpdate _ _ _ = return Logit
instance (Monad m, KnownNat i) => Layer m Logit ('D1 i) ('D1 i) where
runForwards _ (S1D' y) = return $ S1D' (logistic y)
runBackards _ _ (S1D' y) (S1D' dEdy) = return (Logit, S1D' (logistic' y * dEdy))
runBackards _ (S1D' y) (S1D' dEdy) = return ((), S1D' (logistic' y * dEdy))
instance (Monad m, KnownNat i, KnownNat j) => Layer m Logit ('D2 i j) ('D2 i j) where
runForwards _ (S2D' y) = return $ S2D' (logistic y)
runBackards _ _ (S2D' y) (S2D' dEdy) = return (Logit, S2D' (logistic' y * dEdy))
runBackards _ (S2D' y) (S2D' dEdy) = return ((), S2D' (logistic' y * dEdy))
instance (Monad m, KnownNat i, KnownNat j, KnownNat k) => Layer m Logit ('D3 i j k) ('D3 i j k) where
runForwards _ (S3D' y) = return $ S3D' (fmap logistic y)
runBackards _ _ (S3D' y) (S3D' dEdy) = return (Logit, S3D' (vectorZip (\y' dEdy' -> logistic' y' * dEdy') y dEdy))
runBackards _ (S3D' y) (S3D' dEdy) = return ((), S3D' (vectorZip (\y' dEdy' -> logistic' y' * dEdy') y dEdy))
logistic :: Floating a => a -> a

10
src/Grenade/Layers/Pad.hs
View File

@ -1,7 +1,5 @@
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE GADTs #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
@ -39,6 +37,10 @@ data Pad :: Nat
instance Show (Pad padLeft padTop padRight padBottom) where
show Pad = "Pad"
instance Monad m => UpdateLayer m (Pad l t r b) where
type Gradient (Pad l t r b) = ()
runUpdate _ x _ = return x
-- | A two dimentional image can be padped.
instance ( Monad m
, KnownNat padLeft
@ -60,11 +62,11 @@ instance ( Monad m
m = extract input
r = diagBlock [konst 0 (padt,padl), m, konst 0 (padb,padr)]
in return . S2D' . fromJust . create $ r
runBackards _ pad _ (S2D' dEdy) =
runBackards Pad _ (S2D' dEdy) =
let padl = fromIntegral $ natVal (Proxy :: Proxy padLeft)
padt = fromIntegral $ natVal (Proxy :: Proxy padTop)
nrows = fromIntegral $ natVal (Proxy :: Proxy inputRows)
ncols = fromIntegral $ natVal (Proxy :: Proxy inputColumns)
m = extract dEdy
vs = subMatrix (padt, padl) (nrows, ncols) m
in return (pad, S2D' . fromJust . create $ vs)
in return ((), S2D' . fromJust . create $ vs)

12
src/Grenade/Layers/Pooling.hs
View File

@ -51,6 +51,10 @@ instance Show (Pooling k k' s s') where
show Pooling = "Pooling"
instance Monad m => UpdateLayer m (Pooling kernelRows kernelColumns strideRows strideColumns) where
type Gradient (Pooling kr kc sr sc) = ()
runUpdate _ Pooling _ = return Pooling
-- | A two dimentional image can be pooled.
instance ( Monad m
, KnownNat kernelRows
@ -75,7 +79,7 @@ instance ( Monad m
r = poolForward kx ky sx sy ox oy $ ex
rs = fromJust . create $ r
in return . S2D' $ rs
runBackards _ Pooling (S2D' input) (S2D' dEdy) =
runBackards Pooling (S2D' input) (S2D' dEdy) =
let kx = fromIntegral $ natVal (Proxy :: Proxy kernelRows)
ky = fromIntegral $ natVal (Proxy :: Proxy kernelColumns)
sx = fromIntegral $ natVal (Proxy :: Proxy strideRows)
@ -83,7 +87,7 @@ instance ( Monad m
ex = extract input
eo = extract dEdy
vs = poolBackward kx ky sx sy ex eo
in return (Pooling, S2D' . fromJust . create $ vs)
in return ((), S2D' . fromJust . create $ vs)
-- | A three dimensional image can be pooled on each layer.
@ -112,7 +116,7 @@ instance ( Monad m
r = poolForwardList kx ky sx sy ix iy ox oy ex
rs = fmap (fromJust . create) r
in return . S3D' $ rs
runBackards _ Pooling (S3D' input) (S3D' dEdy) =
runBackards Pooling (S3D' input) (S3D' dEdy) =
let ix = fromIntegral $ natVal (Proxy :: Proxy inputRows)
iy = fromIntegral $ natVal (Proxy :: Proxy inputColumns)
kx = fromIntegral $ natVal (Proxy :: Proxy kernelRows)
@ -123,7 +127,7 @@ instance ( Monad m
eo = fmap extract dEdy
ez = vectorZip (,) ex eo
vs = poolBackwardList kx ky sx sy ix iy ez
in return (Pooling, S3D' . fmap (fromJust . create) $ vs)
in return ((), S3D' . fmap (fromJust . create) $ vs)
poolForward :: Int -> Int -> Int -> Int -> Int -> Int -> Matrix Double -> Matrix Double
poolForward nrows ncols srows scols outputRows outputCols m =

12
src/Grenade/Layers/Relu.hs
View File

@ -1,7 +1,5 @@
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE MultiParamTypeClasses #-}
@ -24,11 +22,15 @@ import qualified Numeric.LinearAlgebra.Static as LAS
data Relu = Relu
deriving Show
instance Monad m => UpdateLayer m Relu where
type Gradient Relu = ()
runUpdate _ _ _ = return Relu
instance (Monad m, KnownNat i) => Layer m Relu ('D1 i) ('D1 i) where
runForwards _ (S1D' y) = return $ S1D' (relu y)
where
relu = LAS.dvmap (\a -> if a <= 0 then 0 else a)
runBackards _ _ (S1D' y) (S1D' dEdy) = return (Relu, S1D' (relu' y * dEdy))
runBackards _ (S1D' y) (S1D' dEdy) = return ((), S1D' (relu' y * dEdy))
where
relu' = LAS.dvmap (\a -> if a <= 0 then 0 else 1)
@ -36,7 +38,7 @@ instance (Monad m, KnownNat i, KnownNat j) => Layer m Relu ('D2 i j) ('D2 i j) w
runForwards _ (S2D' y) = return $ S2D' (relu y)
where
relu = LAS.dmmap (\a -> if a <= 0 then 0 else a)
runBackards _ _ (S2D' y) (S2D' dEdy) = return (Relu, S2D' (relu' y * dEdy))
runBackards _ (S2D' y) (S2D' dEdy) = return ((), S2D' (relu' y * dEdy))
where
relu' = LAS.dmmap (\a -> if a <= 0 then 0 else 1)
@ -44,6 +46,6 @@ instance (Monad m, KnownNat i, KnownNat j, KnownNat k) => Layer m Relu ('D3 i j
runForwards _ (S3D' y) = return $ S3D' (fmap relu y)
where
relu = LAS.dmmap (\a -> if a <= 0 then 0 else a)
runBackards _ _ (S3D' y) (S3D' dEdy) = return (Relu, S3D' (vectorZip (\y' dEdy' -> relu' y' * dEdy') y dEdy))
runBackards _ (S3D' y) (S3D' dEdy) = return ((), S3D' (vectorZip (\y' dEdy' -> relu' y' * dEdy') y dEdy))
where
relu' = LAS.dmmap (\a -> if a <= 0 then 0 else 1)

12
src/Grenade/Layers/Tanh.hs
View File

@ -1,7 +1,5 @@
{-# LANGUAGE BangPatterns #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE StandaloneDeriving #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TypeFamilies #-}
{-# LANGUAGE MultiParamTypeClasses #-}
@ -21,17 +19,21 @@ import Grenade.Core.Shape
data Tanh = Tanh
deriving Show
instance Monad m => UpdateLayer m Tanh where
type Gradient Tanh = ()
runUpdate _ _ _ = return Tanh
instance (Monad m, KnownNat i) => Layer m Tanh ('D1 i) ('D1 i) where
runForwards _ (S1D' y) = return $ S1D' (tanh y)
runBackards _ _ (S1D' y) (S1D' dEdy) = return (Tanh, S1D' (tanh' y * dEdy))
runBackards _ (S1D' y) (S1D' dEdy) = return ((), S1D' (tanh' y * dEdy))
instance (Monad m, KnownNat i, KnownNat j) => Layer m Tanh ('D2 i j) ('D2 i j) where
runForwards _ (S2D' y) = return $ S2D' (tanh y)
runBackards _ _ (S2D' y) (S2D' dEdy) = return (Tanh, S2D' (tanh' y * dEdy))
runBackards _ (S2D' y) (S2D' dEdy) = return ((), S2D' (tanh' y * dEdy))
instance (Monad m, KnownNat i, KnownNat j, KnownNat k) => Layer m Tanh ('D3 i j k) ('D3 i j k) where
runForwards _ (S3D' y) = return $ S3D' (fmap tanh y)
runBackards _ _ (S3D' y) (S3D' dEdy) = return (Tanh, S3D' (vectorZip (\y' dEdy' -> tanh' y' * dEdy') y dEdy))
runBackards _ (S3D' y) (S3D' dEdy) = return ((), S3D' (vectorZip (\y' dEdy' -> tanh' y' * dEdy') y dEdy))
tanh' :: (Floating a) => a -> a
tanh' t = 1 - s ^ (2 :: Int) where s = tanh t