@@ -1,4 +1,3 @@
2{-# LANGUAGE DataKinds #-}
1{-# LANGUAGE DataKinds #-}
3{-# LANGUAGE GADTs #-}
2{-# LANGUAGE GADTs #-}
4{-# LANGUAGE TypeOperators #-}
3{-# LANGUAGE TypeOperators #-}
@@ -18,34 +17,37 @@ module Grenade.Recurrent.Core.Network (
19 , RecurrentNetwork (..)
18 , RecurrentNetwork (..)
20 , RecurrentInputs (..)
19 , RecurrentInputs (..)
21− , RecurrentTape (..)
20+ , RecurrentTapes (..)
22− , RecurrentGradient (..)
21+ , RecurrentGradients (..)
24 , randomRecurrent
23 , randomRecurrent
25− , runRecurrent
24+ , runRecurrentNetwork
26− , runRecurrent'
25+ , runRecurrentGradient
27 , applyRecurrentUpdate
26 , applyRecurrentUpdate
31import Control.Monad.Random ( MonadRandom )
30import Control.Monad.Random ( MonadRandom )
31+import Data.Singletons ( SingI )
32+import Data.Singletons.Prelude ( Head, Last )
32import Data.Serialize
33import Data.Serialize
33−
34+import qualified Data.Vector.Storable as V
34−import Data.Kind (Type)
36import Grenade.Core
36import Grenade.Core
37import Grenade.Recurrent.Core.Layer
37import Grenade.Recurrent.Core.Layer
38−import Prelude.Singletons
38+
39+import qualified Numeric.LinearAlgebra as LA
40+import qualified Numeric.LinearAlgebra.Static as LAS
40-- | Witness type to say indicate we're building up with a normal feed
42-- | Witness type to say indicate we're building up with a normal feed
41-- forward layer.
43-- forward layer.
42−data FeedForward :: Type -> Type
44+data FeedForward :: * -> *
43-- | Witness type to say indicate we're building up with a recurrent layer.
45-- | Witness type to say indicate we're building up with a recurrent layer.
44−data Recurrent :: Type -> Type
46+data Recurrent :: * -> *
46-- | Type of a recurrent neural network.
48-- | Type of a recurrent neural network.
48−-- The [Type] type specifies the types of the layers.
50+-- The [*] type specifies the types of the layers.
50-- The [Shape] type specifies the shapes of data passed between the layers.
52-- The [Shape] type specifies the shapes of data passed between the layers.
@@ -54,7 +56,7 @@ data Recurrent :: Type -> Type
55-- Often, to make the definitions more concise, one will use a type alias
57-- Often, to make the definitions more concise, one will use a type alias
56-- for these empty data types.
58-- for these empty data types.
57−data RecurrentNetwork :: [Type] -> [Shape] -> Type where
59+data RecurrentNetwork :: [*] -> [Shape] -> * where
58 RNil :: SingI i
60 RNil :: SingI i
59 => RecurrentNetwork '[] '[i]
61 => RecurrentNetwork '[] '[i]
@@ -73,82 +75,96 @@ infixr 5 :~@>
73-- | Gradient of a network.
75-- | Gradient of a network.
75-- Parameterised on the layers of the network.
77-- Parameterised on the layers of the network.
76−data RecurrentGradient :: [Type] -> Type where
78+data RecurrentGradients :: [*] -> * where
77− RGNil :: RecurrentGradient '[]
79+ RGNil :: RecurrentGradients '[]
79 (://>) :: UpdateLayer x
81 (://>) :: UpdateLayer x
80− => Gradient x
82+ => [Gradient x]
81− -> RecurrentGradient xs
83+ -> RecurrentGradients xs
82− -> RecurrentGradient (phantom x ': xs)
84+ -> RecurrentGradients (phantom x ': xs)
84-- | Recurrent inputs (sideways shapes on an imaginary unrolled graph)
86-- | Recurrent inputs (sideways shapes on an imaginary unrolled graph)
85-- Parameterised on the layers of a Network.
87-- Parameterised on the layers of a Network.
86−data RecurrentInputs :: [Type] -> Type where
88+data RecurrentInputs :: [*] -> * where
87 RINil :: RecurrentInputs '[]
89 RINil :: RecurrentInputs '[]
89− (:~~+>) :: (UpdateLayer x, Fractional (RecurrentInputs xs))
91+ (:~~+>) :: UpdateLayer x
90 => () -> !(RecurrentInputs xs) -> RecurrentInputs (FeedForward x ': xs)
92 => () -> !(RecurrentInputs xs) -> RecurrentInputs (FeedForward x ': xs)
92− (:~@+>) :: (Fractional (RecurrentShape x), Fractional (RecurrentInputs xs), RecurrentUpdateLayer x)
94+ (:~@+>) :: (SingI (RecurrentShape x), RecurrentUpdateLayer x)
93− => !(RecurrentShape x) -> !(RecurrentInputs xs) -> RecurrentInputs (Recurrent x ': xs)
95+ => !(S (RecurrentShape x)) -> !(RecurrentInputs xs) -> RecurrentInputs (Recurrent x ': xs)
95-- | All the information required to backpropogate
97-- | All the information required to backpropogate
96-- through time safely.
98-- through time safely.
98-- We index on the time step length as well, to ensure
100-- We index on the time step length as well, to ensure
99-- that that all Tape lengths are the same.
101-- that that all Tape lengths are the same.
100−data RecurrentTape :: [Type] -> [Shape] -> Type where
102+data RecurrentTapes :: [*] -> [Shape] -> * where
101 TRNil :: SingI i
103 TRNil :: SingI i
102− => RecurrentTape '[] '[i]
104+ => RecurrentTapes '[] '[i]
106+ (:\~>) :: [Tape x i h]
107+ -> !(RecurrentTapes xs (h ': hs))
108+ -> RecurrentTapes (FeedForward x ': xs) (i ': h ': hs)
104− (:\~>) :: Tape x i h
105− -> !(RecurrentTape xs (h ': hs))
106− -> RecurrentTape (FeedForward x ': xs) (i ': h ': hs)
108− (:\@>) :: RecTape x i h
111+ (:\@>) :: [RecTape x i h]
109− -> !(RecurrentTape xs (h ': hs))
112+ -> !(RecurrentTapes xs (h ': hs))
110− -> RecurrentTape (Recurrent x ': xs) (i ': h ': hs)
113+ -> RecurrentTapes (Recurrent x ': xs) (i ': h ': hs)
113−runRecurrent :: forall shapes layers.
116+runRecurrentNetwork :: forall shapes layers.
114− RecurrentNetwork layers shapes
117+ RecurrentNetwork layers shapes
115− -> RecurrentInputs layers
118+ -> RecurrentInputs layers
116− -> S (Head shapes)
119+ -> [S (Head shapes)]
117− -> (RecurrentTape layers shapes, RecurrentInputs layers, S (Last shapes))
120+ -> (RecurrentTapes layers shapes, RecurrentInputs layers, [S (Last shapes)])
118−runRecurrent =
121+runRecurrentNetwork =
121 go :: forall js sublayers. (Last js ~ Last shapes)
124 go :: forall js sublayers. (Last js ~ Last shapes)
122 => RecurrentNetwork sublayers js
125 => RecurrentNetwork sublayers js
123 -> RecurrentInputs sublayers
126 -> RecurrentInputs sublayers
124− -> S (Head js)
127+ -> [S (Head js)]
125− -> (RecurrentTape sublayers js, RecurrentInputs sublayers, S (Last js))
128+ -> (RecurrentTapes sublayers js, RecurrentInputs sublayers, [S (Last js)])
129+ -- This is a simple non-recurrent layer, just map it forwards
126− go (!layer :~~> n) (() :~~+> nIn) !x
130+ go (layer :~~> n) (() :~~+> nIn) !xs
127− = let (!tape, !forwards) = runForwards layer x
131+ = let tys = runForwards layer <$> xs
128−
132+ feedForwardTapes = fst <$> tys
133+ forwards = snd <$> tys
129 -- recursively run the rest of the network, and get the gradients from above.
134 -- recursively run the rest of the network, and get the gradients from above.
130− (!newFN, !ig, !answer) = go n nIn forwards
135+ (newFN, ig, answer) = go n nIn forwards
131− in (tape :\~> newFN, () :~~+> ig, answer)
136+ in (feedForwardTapes :\~> newFN, () :~~+> ig, answer)
133 -- This is a recurrent layer, so we need to do a scan, first input to last, providing
138 -- This is a recurrent layer, so we need to do a scan, first input to last, providing
134 -- the recurrent shape output to the next layer.
139 -- the recurrent shape output to the next layer.
135− go (layer :~@> n) (recIn :~@+> nIn) !x
140+ go (layer :~@> n) (recIn :~@+> nIn) !xs
136− = let (tape, shape, forwards) = runRecurrentForwards layer recIn x
141+ = let (recOut, tys) = goR layer recIn xs
142+ recurrentTapes = fst <$> tys
143+ forwards = snd <$> tys
137 (newFN, ig, answer) = go n nIn forwards
145 (newFN, ig, answer) = go n nIn forwards
138− in (tape :\@> newFN, shape :~@+> ig, answer)
146+ in (recurrentTapes :\@> newFN, recOut :~@+> ig, answer)
140 -- Handle the output layer, bouncing the derivatives back down.
148 -- Handle the output layer, bouncing the derivatives back down.
141 -- We may not have a target for each example, so when we don't use 0 gradient.
149 -- We may not have a target for each example, so when we don't use 0 gradient.
142 go RNil RINil !x
150 go RNil RINil !x
143 = (TRNil, RINil, x)
151 = (TRNil, RINil, x)
153+ -- Helper function for recurrent layers
154+ -- Scans over the recurrent direction of the graph.
155+ goR !layer !recShape (x:xs) =
156+ let (tape, lerec, lepush) = runRecurrentForwards layer recShape x
157+ (rems, push) = goR layer lerec xs
158+ in (rems, (tape, lepush) : push)
159+ goR _ rin [] = (rin, [])
145−runRecurrent' :: forall layers shapes.
161+runRecurrentGradient :: forall layers shapes.
146− RecurrentNetwork layers shapes
162+ RecurrentNetwork layers shapes
147− -> RecurrentTape layers shapes
163+ -> RecurrentTapes layers shapes
148− -> RecurrentInputs layers
164+ -> RecurrentInputs layers
149− -> S (Last shapes)
165+ -> [S (Last shapes)]
150− -> (RecurrentGradient layers, RecurrentInputs layers, S (Head shapes))
166+ -> (RecurrentGradients layers, RecurrentInputs layers, [S (Head shapes)])
151−runRecurrent' net tapes r o =
167+runRecurrentGradient net tapes r o =
152 go net tapes r
168 go net tapes r
154 -- We have to be careful regarding the direction of the lists
170 -- We have to be careful regarding the direction of the lists
@@ -156,41 +172,51 @@ runRecurrent' net tapes r o =
156 -- through time.
172 -- through time.
157 go :: forall js ss. (Last js ~ Last shapes)
173 go :: forall js ss. (Last js ~ Last shapes)
158 => RecurrentNetwork ss js
174 => RecurrentNetwork ss js
159− -> RecurrentTape ss js
175+ -> RecurrentTapes ss js
160 -> RecurrentInputs ss
176 -> RecurrentInputs ss
161− -> (RecurrentGradient ss, RecurrentInputs ss, S (Head js))
177+ -> (RecurrentGradients ss, RecurrentInputs ss, [S (Head js)])
162 -- This is a simple non-recurrent layer
178 -- This is a simple non-recurrent layer
163− -- Run the rest of the network, update with the tapes and gradients
179+ -- Run the rest of the network, then fmap the tapes and gradients
164− go (!layer :~~> n) (!tape :\~> nTapes) (() :~~+> nRecs) =
180+ go (layer :~~> n) (feedForwardTapes :\~> nTapes) (() :~~+> nRecs) =
165− let (!gradients, !rins, !feed) = go n nTapes nRecs
181+ let (gradients, rins, feed) = go n nTapes nRecs
166− (!grad, !back) = runBackwards layer tape feed
182+ backs = uncurry (runBackwards layer) <$> zip (reverse feedForwardTapes) feed
167− in (grad ://> gradients, () :~~+> rins, back)
183+ in ((fst <$> backs) ://> gradients, () :~~+> rins, snd <$> backs)
169 -- This is a recurrent layer
185 -- This is a recurrent layer
170 -- Run the rest of the network, scan over the tapes in reverse
186 -- Run the rest of the network, scan over the tapes in reverse
171− go (!layer :~@> n) (!tape :\@> nTapes) (!recGrad :~@+> nRecs) =
187+ go (layer :~@> n) (recurrentTapes :\@> nTapes) (recGrad :~@+> nRecs) =
172− let (!gradients, !rins, !feed) = go n nTapes nRecs
188+ let (gradients, rins, feed) = go n nTapes nRecs
173− (!grad, !sidegrad, !back) = runRecurrentBackwards layer tape recGrad feed
189+ backExamples = zip (reverse recurrentTapes) feed
190+ (rg, backs) = goX layer recGrad backExamples
174− in (grad ://> gradients, sidegrad :~@+> rins, back)
191+ in ((fst <$> backs) ://> gradients, rg :~@+> rins, snd <$> backs)
176 -- End of the road, so we reflect the given gradients backwards.
193 -- End of the road, so we reflect the given gradients backwards.
177 -- Crucially, we reverse the list, so it's backwards in time as
194 -- Crucially, we reverse the list, so it's backwards in time as
179− go !RNil !TRNil !RINil
196+ go RNil TRNil RINil
180− = (RGNil, RINil, o)
197+ = (RGNil, RINil, reverse o)
199+ -- Helper function for recurrent layers
200+ -- Scans over the recurrent direction of the graph.
201+ goX :: RecurrentLayer x i o => x -> S (RecurrentShape x) -> [(RecTape x i o, S o)] -> (S (RecurrentShape x), [(Gradient x, S i)])
202+ goX layer !lastback ((recTape, backgrad):xs) =
203+ let (layergrad, recgrad, ingrad) = runRecurrentBackwards layer recTape lastback backgrad
204+ (pushedback, ll) = goX layer recgrad xs
205+ in (pushedback, (layergrad, ingrad) : ll)
206+ goX _ !lastback [] = (lastback, [])
182-- | Apply a batch of gradients to the network
208-- | Apply a batch of gradients to the network
183-- Uses runUpdates which can be specialised for
209-- Uses runUpdates which can be specialised for
184-- a layer.
210-- a layer.
185applyRecurrentUpdate :: LearningParameters
211applyRecurrentUpdate :: LearningParameters
186 -> RecurrentNetwork layers shapes
212 -> RecurrentNetwork layers shapes
187− -> RecurrentGradient layers
213+ -> RecurrentGradients layers
188 -> RecurrentNetwork layers shapes
214 -> RecurrentNetwork layers shapes
189applyRecurrentUpdate rate (layer :~~> rest) (gradient ://> grest)
215applyRecurrentUpdate rate (layer :~~> rest) (gradient ://> grest)
190− = runUpdate rate layer gradient :~~> applyRecurrentUpdate rate rest grest
216+ = runUpdates rate layer gradient :~~> applyRecurrentUpdate rate rest grest
192applyRecurrentUpdate rate (layer :~@> rest) (gradient ://> grest)
218applyRecurrentUpdate rate (layer :~@> rest) (gradient ://> grest)
193− = runUpdate rate layer gradient :~@> applyRecurrentUpdate rate rest grest
219+ = runUpdates rate layer gradient :~@> applyRecurrentUpdate rate rest grest
195applyRecurrentUpdate _ RNil RGNil
221applyRecurrentUpdate _ RNil RGNil
@@ -206,25 +232,26 @@ instance (Show x, Show (RecurrentNetwork xs rs)) => Show (RecurrentNetwork (Recu
207-- | A network can easily be created by hand with (:~~>) and (:~@>), but an easy way to initialise a random
233-- | A network can easily be created by hand with (:~~>) and (:~@>), but an easy way to initialise a random
208-- recurrent network and a set of random inputs for it is with the randomRecurrent.
234-- recurrent network and a set of random inputs for it is with the randomRecurrent.
209−class CreatableRecurrent (xs :: [Type]) (ss :: [Shape]) where
235+class CreatableRecurrent (xs :: [*]) (ss :: [Shape]) where
210 -- | Create a network of the types requested
236 -- | Create a network of the types requested
211− randomRecurrent :: MonadRandom m => m (RecurrentNetwork xs ss)
237+ randomRecurrent :: MonadRandom m => m (RecurrentNetwork xs ss, RecurrentInputs xs)
213instance SingI i => CreatableRecurrent '[] '[i] where
239instance SingI i => CreatableRecurrent '[] '[i] where
214 randomRecurrent =
240 randomRecurrent =
215− return RNil
241+ return (RNil, RINil)
217instance (SingI i, Layer x i o, CreatableRecurrent xs (o ': rs)) => CreatableRecurrent (FeedForward x ': xs) (i ': o ': rs) where
243instance (SingI i, Layer x i o, CreatableRecurrent xs (o ': rs)) => CreatableRecurrent (FeedForward x ': xs) (i ': o ': rs) where
218 randomRecurrent = do
244 randomRecurrent = do
219 thisLayer <- createRandom
245 thisLayer <- createRandom
220− rest <- randomRecurrent
246+ (rest, resti) <- randomRecurrent
221− return (thisLayer :~~> rest)
247+ return (thisLayer :~~> rest, () :~~+> resti)
223instance (SingI i, RecurrentLayer x i o, CreatableRecurrent xs (o ': rs)) => CreatableRecurrent (Recurrent x ': xs) (i ': o ': rs) where
249instance (SingI i, RecurrentLayer x i o, CreatableRecurrent xs (o ': rs)) => CreatableRecurrent (Recurrent x ': xs) (i ': o ': rs) where
224 randomRecurrent = do
250 randomRecurrent = do
225 thisLayer <- createRandom
251 thisLayer <- createRandom
252+ thisShape <- randomOfShape
226− rest <- randomRecurrent
253+ (rest, resti) <- randomRecurrent
227− return (thisLayer :~@> rest)
254+ return (thisLayer :~@> rest, thisShape :~@+> resti)
229-- | Add very simple serialisation to the recurrent network
256-- | Add very simple serialisation to the recurrent network
230instance SingI i => Serialize (RecurrentNetwork '[] '[i]) where
257instance SingI i => Serialize (RecurrentNetwork '[] '[i]) where
@@ -243,13 +270,23 @@ instance (Serialize (RecurrentInputs '[])) where
243 put _ = return ()
270 put _ = return ()
244 get = return RINil
271 get = return RINil
246−instance (UpdateLayer x, Serialize (RecurrentInputs ys), Fractional (RecurrentInputs ys)) => (Serialize (RecurrentInputs (FeedForward x ': ys))) where
273+instance (UpdateLayer x, Serialize (RecurrentInputs ys)) => (Serialize (RecurrentInputs (FeedForward x ': ys))) where
247 put ( () :~~+> rest) = put rest
274 put ( () :~~+> rest) = put rest
248 get = ( () :~~+> ) <$> get
275 get = ( () :~~+> ) <$> get
250−instance (Serialize (RecurrentShape x), Fractional (RecurrentShape x), RecurrentUpdateLayer x, Serialize (RecurrentInputs ys), Fractional (RecurrentInputs ys)) => (Serialize (RecurrentInputs (Recurrent x ': ys))) where
277+instance (SingI (RecurrentShape x), RecurrentUpdateLayer x, Serialize (RecurrentInputs ys)) => (Serialize (RecurrentInputs (Recurrent x ': ys))) where
251− put ( i :~@+> rest ) = put i >> put rest
278+ put ( i :~@+> rest ) = do
280+ (S1D x) -> putListOf put . LA.toList . LAS.extract $ x
281+ (S2D x) -> putListOf put . LA.toList . LA.flatten . LAS.extract $ x
282+ (S3D x) -> putListOf put . LA.toList . LA.flatten . LAS.extract $ x
252− get = (:~@+>) <$> get <*> get
286+ get = do
287+ Just i <- fromStorable . V.fromList <$> getListOf get
289+ return ( i :~@+> rest)
255-- Num instance for `RecurrentInputs layers`
292-- Num instance for `RecurrentInputs layers`
@@ -270,7 +307,7 @@ instance (Num (RecurrentInputs '[])) where
270 signum _ = RINil
307 signum _ = RINil
271 fromInteger _ = RINil
308 fromInteger _ = RINil
273−instance (UpdateLayer x, Fractional (RecurrentInputs ys)) => (Num (RecurrentInputs (FeedForward x ': ys))) where
310+instance (UpdateLayer x, Num (RecurrentInputs ys)) => (Num (RecurrentInputs (FeedForward x ': ys))) where
274 (+) (() :~~+> x) (() :~~+> y) = () :~~+> (x + y)
311 (+) (() :~~+> x) (() :~~+> y) = () :~~+> (x + y)
275 (-) (() :~~+> x) (() :~~+> y) = () :~~+> (x - y)
312 (-) (() :~~+> x) (() :~~+> y) = () :~~+> (x - y)
276 (*) (() :~~+> x) (() :~~+> y) = () :~~+> (x * y)
313 (*) (() :~~+> x) (() :~~+> y) = () :~~+> (x * y)
@@ -278,51 +315,10 @@ instance (UpdateLayer x, Fractional (RecurrentInputs ys)) => (Num (RecurrentInpu
278 signum (() :~~+> x) = () :~~+> signum x
315 signum (() :~~+> x) = () :~~+> signum x
279 fromInteger x = () :~~+> fromInteger x
316 fromInteger x = () :~~+> fromInteger x
281−instance (Fractional (RecurrentShape x), RecurrentUpdateLayer x, Fractional (RecurrentInputs ys)) => (Num (RecurrentInputs (Recurrent x ': ys))) where
318+instance (SingI (RecurrentShape x), RecurrentUpdateLayer x, Num (RecurrentInputs ys)) => (Num (RecurrentInputs (Recurrent x ': ys))) where
282 (+) (x :~@+> x') (y :~@+> y') = (x + y) :~@+> (x' + y')
319 (+) (x :~@+> x') (y :~@+> y') = (x + y) :~@+> (x' + y')
283 (-) (x :~@+> x') (y :~@+> y') = (x - y) :~@+> (x' - y')
320 (-) (x :~@+> x') (y :~@+> y') = (x - y) :~@+> (x' - y')
284 (*) (x :~@+> x') (y :~@+> y') = (x * y) :~@+> (x' * y')
321 (*) (x :~@+> x') (y :~@+> y') = (x * y) :~@+> (x' * y')
285 abs (x :~@+> x') = abs x :~@+> abs x'
322 abs (x :~@+> x') = abs x :~@+> abs x'
286 signum (x :~@+> x') = signum x :~@+> signum x'
323 signum (x :~@+> x') = signum x :~@+> signum x'
287 fromInteger x = fromInteger x :~@+> fromInteger x
324 fromInteger x = fromInteger x :~@+> fromInteger x
289−instance (Fractional (RecurrentInputs '[])) where
292− fromRational _ = RINil
294−instance (UpdateLayer x, Fractional (RecurrentInputs ys)) => (Fractional (RecurrentInputs (FeedForward x ': ys))) where
295− (/) (() :~~+> x) (() :~~+> y) = () :~~+> (x / y)
296− recip (() :~~+> x) = () :~~+> recip x
297− fromRational x = () :~~+> fromRational x
299−instance (Fractional (RecurrentShape x), RecurrentUpdateLayer x, Fractional (RecurrentInputs ys)) => (Fractional (RecurrentInputs (Recurrent x ': ys))) where
300− (/) (x :~@+> x') (y :~@+> y') = (x / y) :~@+> (x' / y')
301− recip (x :~@+> x') = recip x :~@+> recip x'
302− fromRational x = fromRational x :~@+> fromRational x
305−-- | Ultimate composition.
307−-- This allows a complete network to be treated as a layer in a larger network.
308−instance CreatableRecurrent sublayers subshapes => UpdateLayer (RecurrentNetwork sublayers subshapes) where
309− type Gradient (RecurrentNetwork sublayers subshapes) = RecurrentGradient sublayers
310− runUpdate = applyRecurrentUpdate
311− createRandom = randomRecurrent
313−-- | Ultimate composition.
315−-- This allows a complete network to be treated as a layer in a larger network.
316−instance CreatableRecurrent sublayers subshapes => RecurrentUpdateLayer (RecurrentNetwork sublayers subshapes) where
317− type RecurrentShape (RecurrentNetwork sublayers subshapes) = RecurrentInputs sublayers
319−-- | Ultimate composition.
321−-- This allows a complete network to be treated as a layer in a larger network.
322−instance ( CreatableRecurrent sublayers subshapes
323− , i ~ (Head subshapes), o ~ (Last subshapes)
324− , Num (RecurrentShape (RecurrentNetwork sublayers subshapes))
325− ) => RecurrentLayer (RecurrentNetwork sublayers subshapes) i o where
326− type RecTape (RecurrentNetwork sublayers subshapes) i o = RecurrentTape sublayers subshapes
327− runRecurrentForwards = runRecurrent
328− runRecurrentBackwards = runRecurrent'