9fd2111 · grenade · git.iko.soy

bench/bench-lstm.hs+2−2

compare with master

unifiedsplit

@@ -27,8 +27,8 @@ main = do

27 , bench "backwards-60" $ nf (nfT3 . uncurry4 (testRun60' layer60)) (rec60, input40, rec60, rec60)

28 , bench "backwards-512" $ nf (nfT3 . uncurry4 (testRun512' layer512)) (rec512, input40, rec512, rec512)

29 ]

30− , bgroup "update" [ bench "matrix-60x60" $ nf (uncurry3 (decendVector 1 1 1)) (upIn60, upIn60, upIn60)

30+ , bgroup "update" [ bench "matrix-60x60" $ nf (uncurry3 (descendVector 1 1 1)) (upIn60, upIn60, upIn60)

31− , bench "matrix-512x512" $ nf (uncurry3 (decendVector 1 1 1)) (upIn512, upIn512, upIn512)

31+ , bench "matrix-512x512" $ nf (uncurry3 (descendVector 1 1 1)) (upIn512, upIn512, upIn512)

32 ]

33 , bgroup "train" [ bench "one-time-step" $ whnf (nfT2 . trainRecurrent lp lstm 0) [(input40, Just input40)]

34 , bench "ten-time-steps" $ whnf (nfT2 . trainRecurrent lp lstm 0) $ replicate 10 (input40, Just input40)

cbits/gradient_decent.c → cbits/gradient_descent.crenamed+2−2

@@ -1,6 +1,6 @@

1−#include "gradient_decent.h"

1+#include "gradient_descent.h"

2

3−void decend_cpu(int len, double rate, double momentum, double regulariser,

3+void descend_cpu(int len, double rate, double momentum, double regulariser,

4 const double* weights,

5 const double* gradient,

6 const double* last,

cbits/gradient_decent.h → cbits/gradient_descent.hrenamed+1−1

@@ -1,7 +1,7 @@

1#include <stdio.h>

2#include <stdint.h>

3

4−void decend_cpu(int len, double rate, double momentum, double regulariser,

4+void descend_cpu(int len, double rate, double momentum, double regulariser,

5 const double* weights,

6 const double* gradient,

7 const double* last,

grenade.cabal+5−5

@@ -17,7 +17,7 @@ description:

17 Grenade provides an API for composing layers of a neural network

18 into a sequence parallel graph in a type safe manner; running

19 networks with reverse automatic differentiation to calculate their

20− gradients; and applying gradient decent for learning.

20+ gradients; and applying gradient descent for learning.

21 .

22 Documentation and examples are available on github

23 <https://github.com/HuwCampbell/grenade>.

@@ -26,8 +26,8 @@ extra-source-files:

26 README.md

27 cbits/im2col.h

28 cbits/im2col.c

29− cbits/gradient_decent.h

29+ cbits/gradient_descent.h

30− cbits/gradient_decent.c

30+ cbits/gradient_descent.c

31 cbits/pad.h

32 cbits/pad.c

33

@@ -108,10 +108,10 @@ library

108 Grenade.Utils.OneHot

109

110 includes: cbits/im2col.h

111− cbits/gradient_decent.h

111+ cbits/gradient_descent.h

112 cbits/pad.h

113 c-sources: cbits/im2col.c

114− cbits/gradient_decent.c

114+ cbits/gradient_descent.c

115 cbits/pad.c

116

117 cc-options: -std=c99 -O3 -msse4.2 -Wall -Werror -DCABAL=1

src/Grenade/Core/Network.hs+1−1

@@ -139,7 +139,7 @@ runGradient net tapes o =

139 = (GNil, o)

140

141

142−-- | Apply one step of stochastic gradient decent across the network.

142+-- | Apply one step of stochastic gradient descent across the network.

143applyUpdate :: LearningParameters

144 -> Network layers shapes

145 -> Gradients layers

src/Grenade/Layers/Convolution.hs+1−1

@@ -131,7 +131,7 @@ instance ( KnownNat channels

131 ) => UpdateLayer (Convolution channels filters kernelRows kernelColumns strideRows strideColumns) where

132 type Gradient (Convolution channels filters kernelRows kernelCols strideRows strideCols) = (Convolution' channels filters kernelRows kernelCols strideRows strideCols)

133 runUpdate LearningParameters {..} (Convolution oldKernel oldMomentum) (Convolution' kernelGradient) =

134− let (newKernel, newMomentum) = decendMatrix learningRate learningMomentum learningRegulariser oldKernel kernelGradient oldMomentum

134+ let (newKernel, newMomentum) = descendMatrix learningRate learningMomentum learningRegulariser oldKernel kernelGradient oldMomentum

135 in Convolution newKernel newMomentum

136

137 createRandom = randomConvolution

src/Grenade/Layers/Deconvolution.hs+1−1

@@ -130,7 +130,7 @@ instance ( KnownNat channels

130 ) => UpdateLayer (Deconvolution channels filters kernelRows kernelColumns strideRows strideColumns) where

131 type Gradient (Deconvolution channels filters kernelRows kernelCols strideRows strideCols) = (Deconvolution' channels filters kernelRows kernelCols strideRows strideCols)

132 runUpdate LearningParameters {..} (Deconvolution oldKernel oldMomentum) (Deconvolution' kernelGradient) =

133− let (newKernel, newMomentum) = decendMatrix learningRate learningMomentum learningRegulariser oldKernel kernelGradient oldMomentum

133+ let (newKernel, newMomentum) = descendMatrix learningRate learningMomentum learningRegulariser oldKernel kernelGradient oldMomentum

134 in Deconvolution newKernel newMomentum

135

136 createRandom = randomDeconvolution

src/Grenade/Layers/FullyConnected.hs+2−2

@@ -39,8 +39,8 @@ instance (KnownNat i, KnownNat o) => UpdateLayer (FullyConnected i o) where

39 type Gradient (FullyConnected i o) = (FullyConnected' i o)

40

41 runUpdate LearningParameters {..} (FullyConnected (FullyConnected' oldBias oldActivations) (FullyConnected' oldBiasMomentum oldMomentum)) (FullyConnected' biasGradient activationGradient) =

42− let (newBias, newBiasMomentum) = decendVector learningRate learningMomentum learningRegulariser oldBias biasGradient oldBiasMomentum

42+ let (newBias, newBiasMomentum) = descendVector learningRate learningMomentum learningRegulariser oldBias biasGradient oldBiasMomentum

43− (newActivations, newMomentum) = decendMatrix learningRate learningMomentum learningRegulariser oldActivations activationGradient oldMomentum

43+ (newActivations, newMomentum) = descendMatrix learningRate learningMomentum learningRegulariser oldActivations activationGradient oldMomentum

44 in FullyConnected (FullyConnected' newBias newActivations) (FullyConnected' newBiasMomentum newMomentum)

45

46 createRandom = randomFullyConnected

src/Grenade/Layers/Internal/Update.hs+12−12

@@ -1,7 +1,7 @@

1{-# LANGUAGE ForeignFunctionInterface #-}

2module Grenade.Layers.Internal.Update (

3− decendMatrix

3+ descendMatrix

4− , decendVector

4+ , descendVector

5 ) where

6

7import Data.Maybe ( fromJust )

@@ -17,8 +17,8 @@ import qualified Numeric.LinearAlgebra.Devel as U

17

18import System.IO.Unsafe ( unsafePerformIO )

19

20−decendMatrix :: (KnownNat rows, KnownNat columns) => Double -> Double -> Double -> L rows columns -> L rows columns -> L rows columns -> (L rows columns, L rows columns)

20+descendMatrix :: (KnownNat rows, KnownNat columns) => Double -> Double -> Double -> L rows columns -> L rows columns -> L rows columns -> (L rows columns, L rows columns)

21−decendMatrix rate momentum regulariser weights gradient lastUpdate =

21+descendMatrix rate momentum regulariser weights gradient lastUpdate =

22 let (rows, cols) = size weights

23 len = rows * cols

24 -- Most gradients come in in ColumnMajor,

@@ -29,7 +29,7 @@ decendMatrix rate momentum regulariser weights gradient lastUpdate =

29 weights' = flatten . tr . extract $ weights

30 gradient' = flatten . tr . extract $ gradient

31 lastUpdate' = flatten . tr . extract $ lastUpdate

32− (vw, vm) = decendUnsafe len rate momentum regulariser weights' gradient' lastUpdate'

32+ (vw, vm) = descendUnsafe len rate momentum regulariser weights' gradient' lastUpdate'

33

34 -- Note that it's ColumnMajor, as we did a transpose before

35 -- using the internal vectors.

@@ -37,17 +37,17 @@ decendMatrix rate momentum regulariser weights gradient lastUpdate =

37 mm = U.matrixFromVector U.ColumnMajor rows cols vm

38 in (fromJust . create $ mw, fromJust . create $ mm)

39

40−decendVector :: (KnownNat r) => Double -> Double -> Double -> R r -> R r -> R r -> (R r, R r)

40+descendVector :: (KnownNat r) => Double -> Double -> Double -> R r -> R r -> R r -> (R r, R r)

41−decendVector rate momentum regulariser weights gradient lastUpdate =

41+descendVector rate momentum regulariser weights gradient lastUpdate =

42 let len = size weights

43 weights' = extract weights

44 gradient' = extract gradient

45 lastUpdate' = extract lastUpdate

46− (vw, vm) = decendUnsafe len rate momentum regulariser weights' gradient' lastUpdate'

46+ (vw, vm) = descendUnsafe len rate momentum regulariser weights' gradient' lastUpdate'

47 in (fromJust $ create vw, fromJust $ create vm)

48

49−decendUnsafe :: Int -> Double -> Double -> Double -> Vector Double -> Vector Double -> Vector Double -> (Vector Double, Vector Double)

49+descendUnsafe :: Int -> Double -> Double -> Double -> Vector Double -> Vector Double -> Vector Double -> (Vector Double, Vector Double)

50−decendUnsafe len rate momentum regulariser weights gradient lastUpdate =

50+descendUnsafe len rate momentum regulariser weights gradient lastUpdate =

51 unsafePerformIO $ do

52 outWPtr <- mallocForeignPtrArray len

53 outMPtr <- mallocForeignPtrArray len

@@ -60,11 +60,11 @@ decendUnsafe len rate momentum regulariser weights gradient lastUpdate =

60 withForeignPtr lPtr $ \lPtr' ->

61 withForeignPtr outWPtr $ \outWPtr' ->

62 withForeignPtr outMPtr $ \outMPtr' ->

63− decend_cpu len rate momentum regulariser wPtr' gPtr' lPtr' outWPtr' outMPtr'

63+ descend_cpu len rate momentum regulariser wPtr' gPtr' lPtr' outWPtr' outMPtr'

64

65 return (U.unsafeFromForeignPtr0 outWPtr len, U.unsafeFromForeignPtr0 outMPtr len)

66

67foreign import ccall unsafe

68− decend_cpu

68+ descend_cpu

69 :: Int -> Double -> Double -> Double -> Ptr Double -> Ptr Double -> Ptr Double -> Ptr Double -> Ptr Double -> IO ()

70

src/Grenade/Recurrent/Layers/LSTM.hs+2−2

@@ -87,11 +87,11 @@ instance (KnownNat i, KnownNat o) => UpdateLayer (LSTM i o) where

87 -- Utility function for updating with the momentum, gradients, and weights.

88 u :: forall x ix out. (KnownNat ix, KnownNat out) => (x -> (L out ix)) -> x -> x -> x -> ((L out ix), (L out ix))

89 u e (e -> weights) (e -> momentum) (e -> gradient) =

90− decendMatrix learningRate learningMomentum learningRegulariser weights gradient momentum

90+ descendMatrix learningRate learningMomentum learningRegulariser weights gradient momentum

91

92 v :: forall x ix. (KnownNat ix) => (x -> (R ix)) -> x -> x -> x -> ((R ix), (R ix))

93 v e (e -> weights) (e -> momentum) (e -> gradient) =

94− decendVector learningRate learningMomentum learningRegulariser weights gradient momentum

94+ descendVector learningRate learningMomentum learningRegulariser weights gradient momentum

95

96 -- There's a lot of updates here, so to try and minimise the number of data copies

97 -- we'll create a mutable bucket for each.