9fd2111 · grenade · git.iko.soy

git.iko.soy/grenade

Spelling: decend/decent -> descend/descent

commit9fd21111225d56d49eeafc2ef65da138c5a92954 authorErik de Castro Lopo <erikd@mega-nerd.com> dateMon 25 Sep 2017, 19:56 (9 years ago) parent8e3e7db

10 files changed+29−29

compare with master

unifiedsplit

bench/bench-lstm.hs+2−2 cbits/gradient_decent.c → cbits/gradient_descent.c+2−2 cbits/gradient_decent.h → cbits/gradient_descent.h+1−1 grenade.cabal+5−5 src/Grenade/Core/Network.hs+1−1 src/Grenade/Layers/Convolution.hs+1−1 src/Grenade/Layers/Deconvolution.hs+1−1 src/Grenade/Layers/FullyConnected.hs+2−2 src/Grenade/Layers/Internal/Update.hs+12−12 src/Grenade/Recurrent/Layers/LSTM.hs+2−2

bench/bench-lstm.hs+2−2

@@ -27,8 +27,8 @@ main = do

2727 , bench "backwards-60" $ nf (nfT3 . uncurry4 (testRun60' layer60)) (rec60, input40, rec60, rec60)

2828 , bench "backwards-512" $ nf (nfT3 . uncurry4 (testRun512' layer512)) (rec512, input40, rec512, rec512)

2929 ]

30− , bgroup "update" [ bench "matrix-60x60" $ nf (uncurry3 (decendVector 1 1 1)) (upIn60, upIn60, upIn60)

31− , bench "matrix-512x512" $ nf (uncurry3 (decendVector 1 1 1)) (upIn512, upIn512, upIn512)

30+ , bgroup "update" [ bench "matrix-60x60" $ nf (uncurry3 (descendVector 1 1 1)) (upIn60, upIn60, upIn60)

31+ , bench "matrix-512x512" $ nf (uncurry3 (descendVector 1 1 1)) (upIn512, upIn512, upIn512)

3232 ]

3333 , bgroup "train" [ bench "one-time-step" $ whnf (nfT2 . trainRecurrent lp lstm 0) [(input40, Just input40)]

3434 , bench "ten-time-steps" $ whnf (nfT2 . trainRecurrent lp lstm 0) $ replicate 10 (input40, Just input40)

cbits/gradient_decent.c → cbits/gradient_descent.crenamed+2−2

@@ -1,6 +1,6 @@

1−#include "gradient_decent.h"

1+#include "gradient_descent.h"

3−void decend_cpu(int len, double rate, double momentum, double regulariser,

3+void descend_cpu(int len, double rate, double momentum, double regulariser,

44 const double* weights,

55 const double* gradient,

66 const double* last,

cbits/gradient_decent.h → cbits/gradient_descent.hrenamed+1−1

@@ -1,7 +1,7 @@

11#include <stdio.h>

22#include <stdint.h>

4−void decend_cpu(int len, double rate, double momentum, double regulariser,

4+void descend_cpu(int len, double rate, double momentum, double regulariser,

55 const double* weights,

66 const double* gradient,

77 const double* last,

grenade.cabal+5−5

@@ -17,7 +17,7 @@ description:

1717 Grenade provides an API for composing layers of a neural network

1818 into a sequence parallel graph in a type safe manner; running

1919 networks with reverse automatic differentiation to calculate their

20− gradients; and applying gradient decent for learning.

20+ gradients; and applying gradient descent for learning.

2121 .

2222 Documentation and examples are available on github

2323 <https://github.com/HuwCampbell/grenade>.

@@ -26,8 +26,8 @@ extra-source-files:

2626 README.md

2727 cbits/im2col.h

2828 cbits/im2col.c

29− cbits/gradient_decent.h

30− cbits/gradient_decent.c

29+ cbits/gradient_descent.h

30+ cbits/gradient_descent.c

3131 cbits/pad.h

3232 cbits/pad.c

3333

@@ -108,10 +108,10 @@ library

108108 Grenade.Utils.OneHot

109109

110110 includes: cbits/im2col.h

111− cbits/gradient_decent.h

111+ cbits/gradient_descent.h

112112 cbits/pad.h

113113 c-sources: cbits/im2col.c

114− cbits/gradient_decent.c

114+ cbits/gradient_descent.c

115115 cbits/pad.c

116116

117117 cc-options: -std=c99 -O3 -msse4.2 -Wall -Werror -DCABAL=1

src/Grenade/Core/Network.hs+1−1

@@ -139,7 +139,7 @@ runGradient net tapes o =

139139 = (GNil, o)

140140

141141

142−-- | Apply one step of stochastic gradient decent across the network.

142+-- | Apply one step of stochastic gradient descent across the network.

143143applyUpdate :: LearningParameters

144144 -> Network layers shapes

145145 -> Gradients layers

src/Grenade/Layers/Convolution.hs+1−1

@@ -131,7 +131,7 @@ instance ( KnownNat channels

131131 ) => UpdateLayer (Convolution channels filters kernelRows kernelColumns strideRows strideColumns) where

132132 type Gradient (Convolution channels filters kernelRows kernelCols strideRows strideCols) = (Convolution' channels filters kernelRows kernelCols strideRows strideCols)

133133 runUpdate LearningParameters {..} (Convolution oldKernel oldMomentum) (Convolution' kernelGradient) =

134− let (newKernel, newMomentum) = decendMatrix learningRate learningMomentum learningRegulariser oldKernel kernelGradient oldMomentum

134+ let (newKernel, newMomentum) = descendMatrix learningRate learningMomentum learningRegulariser oldKernel kernelGradient oldMomentum

135135 in Convolution newKernel newMomentum

136136

137137 createRandom = randomConvolution

src/Grenade/Layers/Deconvolution.hs+1−1

@@ -130,7 +130,7 @@ instance ( KnownNat channels

130130 ) => UpdateLayer (Deconvolution channels filters kernelRows kernelColumns strideRows strideColumns) where

131131 type Gradient (Deconvolution channels filters kernelRows kernelCols strideRows strideCols) = (Deconvolution' channels filters kernelRows kernelCols strideRows strideCols)

132132 runUpdate LearningParameters {..} (Deconvolution oldKernel oldMomentum) (Deconvolution' kernelGradient) =

133− let (newKernel, newMomentum) = decendMatrix learningRate learningMomentum learningRegulariser oldKernel kernelGradient oldMomentum

133+ let (newKernel, newMomentum) = descendMatrix learningRate learningMomentum learningRegulariser oldKernel kernelGradient oldMomentum

134134 in Deconvolution newKernel newMomentum

135135

136136 createRandom = randomDeconvolution

src/Grenade/Layers/FullyConnected.hs+2−2

@@ -39,8 +39,8 @@ instance (KnownNat i, KnownNat o) => UpdateLayer (FullyConnected i o) where

3939 type Gradient (FullyConnected i o) = (FullyConnected' i o)

4040

4141 runUpdate LearningParameters {..} (FullyConnected (FullyConnected' oldBias oldActivations) (FullyConnected' oldBiasMomentum oldMomentum)) (FullyConnected' biasGradient activationGradient) =

42− let (newBias, newBiasMomentum) = decendVector learningRate learningMomentum learningRegulariser oldBias biasGradient oldBiasMomentum

43− (newActivations, newMomentum) = decendMatrix learningRate learningMomentum learningRegulariser oldActivations activationGradient oldMomentum

42+ let (newBias, newBiasMomentum) = descendVector learningRate learningMomentum learningRegulariser oldBias biasGradient oldBiasMomentum

43+ (newActivations, newMomentum) = descendMatrix learningRate learningMomentum learningRegulariser oldActivations activationGradient oldMomentum

4444 in FullyConnected (FullyConnected' newBias newActivations) (FullyConnected' newBiasMomentum newMomentum)

4545

4646 createRandom = randomFullyConnected

src/Grenade/Layers/Internal/Update.hs+12−12

@@ -1,7 +1,7 @@

11{-# LANGUAGE ForeignFunctionInterface #-}

22module Grenade.Layers.Internal.Update (

3− decendMatrix

4− , decendVector

3+ descendMatrix

4+ , descendVector

55 ) where

77import Data.Maybe ( fromJust )

@@ -17,8 +17,8 @@ import qualified Numeric.LinearAlgebra.Devel as U

1717

1818import System.IO.Unsafe ( unsafePerformIO )

1919

20−decendMatrix :: (KnownNat rows, KnownNat columns) => Double -> Double -> Double -> L rows columns -> L rows columns -> L rows columns -> (L rows columns, L rows columns)

21−decendMatrix rate momentum regulariser weights gradient lastUpdate =

20+descendMatrix :: (KnownNat rows, KnownNat columns) => Double -> Double -> Double -> L rows columns -> L rows columns -> L rows columns -> (L rows columns, L rows columns)

21+descendMatrix rate momentum regulariser weights gradient lastUpdate =

2222 let (rows, cols) = size weights

2323 len = rows * cols

2424 -- Most gradients come in in ColumnMajor,

@@ -29,7 +29,7 @@ decendMatrix rate momentum regulariser weights gradient lastUpdate =

2929 weights' = flatten . tr . extract $ weights

3030 gradient' = flatten . tr . extract $ gradient

3131 lastUpdate' = flatten . tr . extract $ lastUpdate

32− (vw, vm) = decendUnsafe len rate momentum regulariser weights' gradient' lastUpdate'

32+ (vw, vm) = descendUnsafe len rate momentum regulariser weights' gradient' lastUpdate'

3333

3434 -- Note that it's ColumnMajor, as we did a transpose before

3535 -- using the internal vectors.

@@ -37,17 +37,17 @@ decendMatrix rate momentum regulariser weights gradient lastUpdate =

3737 mm = U.matrixFromVector U.ColumnMajor rows cols vm

3838 in (fromJust . create $ mw, fromJust . create $ mm)

3939

40−decendVector :: (KnownNat r) => Double -> Double -> Double -> R r -> R r -> R r -> (R r, R r)

41−decendVector rate momentum regulariser weights gradient lastUpdate =

40+descendVector :: (KnownNat r) => Double -> Double -> Double -> R r -> R r -> R r -> (R r, R r)

41+descendVector rate momentum regulariser weights gradient lastUpdate =

4242 let len = size weights

4343 weights' = extract weights

4444 gradient' = extract gradient

4545 lastUpdate' = extract lastUpdate

46− (vw, vm) = decendUnsafe len rate momentum regulariser weights' gradient' lastUpdate'

46+ (vw, vm) = descendUnsafe len rate momentum regulariser weights' gradient' lastUpdate'

4747 in (fromJust $ create vw, fromJust $ create vm)

4848

49−decendUnsafe :: Int -> Double -> Double -> Double -> Vector Double -> Vector Double -> Vector Double -> (Vector Double, Vector Double)

50−decendUnsafe len rate momentum regulariser weights gradient lastUpdate =

49+descendUnsafe :: Int -> Double -> Double -> Double -> Vector Double -> Vector Double -> Vector Double -> (Vector Double, Vector Double)

50+descendUnsafe len rate momentum regulariser weights gradient lastUpdate =

5151 unsafePerformIO $ do

5252 outWPtr <- mallocForeignPtrArray len

5353 outMPtr <- mallocForeignPtrArray len

@@ -60,11 +60,11 @@ decendUnsafe len rate momentum regulariser weights gradient lastUpdate =

6060 withForeignPtr lPtr $ \lPtr' ->

6161 withForeignPtr outWPtr $ \outWPtr' ->

6262 withForeignPtr outMPtr $ \outMPtr' ->

63− decend_cpu len rate momentum regulariser wPtr' gPtr' lPtr' outWPtr' outMPtr'

63+ descend_cpu len rate momentum regulariser wPtr' gPtr' lPtr' outWPtr' outMPtr'

6464

6565 return (U.unsafeFromForeignPtr0 outWPtr len, U.unsafeFromForeignPtr0 outMPtr len)

6666

6767foreign import ccall unsafe

68− decend_cpu

68+ descend_cpu

6969 :: Int -> Double -> Double -> Double -> Ptr Double -> Ptr Double -> Ptr Double -> Ptr Double -> Ptr Double -> IO ()

7070

src/Grenade/Recurrent/Layers/LSTM.hs+2−2

@@ -87,11 +87,11 @@ instance (KnownNat i, KnownNat o) => UpdateLayer (LSTM i o) where

8787 -- Utility function for updating with the momentum, gradients, and weights.

8888 u :: forall x ix out. (KnownNat ix, KnownNat out) => (x -> (L out ix)) -> x -> x -> x -> ((L out ix), (L out ix))

8989 u e (e -> weights) (e -> momentum) (e -> gradient) =

90− decendMatrix learningRate learningMomentum learningRegulariser weights gradient momentum

90+ descendMatrix learningRate learningMomentum learningRegulariser weights gradient momentum

9191

9292 v :: forall x ix. (KnownNat ix) => (x -> (R ix)) -> x -> x -> x -> ((R ix), (R ix))

9393 v e (e -> weights) (e -> momentum) (e -> gradient) =

94− decendVector learningRate learningMomentum learningRegulariser weights gradient momentum

94+ descendVector learningRate learningMomentum learningRegulariser weights gradient momentum

9595

9696 -- There's a lot of updates here, so to try and minimise the number of data copies

9797 -- we'll create a mutable bucket for each.